13:46:34.867 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
13:46:34.879 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHFxKIK/test-tmp
13:46:34.882 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:46:34.886 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHFxKIK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:46:35.160 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
13:46:35.313 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:46:35.314 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:46:35.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.317559 18011 mini_oidc.cc:152] Starting JWKS server
13:46:35.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.322633 18011 webserver.cc:460] Webserver started at http://127.0.0.1:39703/ using document root <none> and password file <none>
13:46:35.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.323091 18011 mini_oidc.cc:189] Starting OIDC Discovery server
13:46:35.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.325384 18011 webserver.cc:460] Webserver started at http://127.0.0.1:34419/ using document root <none> and password file <none>
13:46:35.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T13:46:35Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:46:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T13:46:35Z Disabled control of system clock
13:46:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.374318 18011 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/wal
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:35.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/logs
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data/info.pb
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:35.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.254:35555
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.254
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:35.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39703/jwks.json with env {}
13:46:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.691206 18023 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.691756 18023 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:35.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.692401 18023 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:35.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.722486 18023 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.722764 18023 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.722949 18023 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.723130 18023 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.723390 18023 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:35.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.723611 18023 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39703/jwks.json
13:46:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.757850 18023 master_runner.cc:386] Master server non-default flags:
13:46:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:35.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data
13:46:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/wal
13:46:35.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.254:35555
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39703/jwks.json
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data/info.pb
13:46:35.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.254
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/logs
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.759227 18023 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.761407 18023 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.784600 18030 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.784610 18029 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:35.787515 18023 server_base.cc:1034] running on GCE node
13:46:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:35.785492 18032 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.905514 18023 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:36.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.908869 18023 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:36.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.910264 18023 hybrid_clock.cc:648] HybridClock initialized: now 1790603196910219 us; error 58 us; skew 500 ppm
13:46:36.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.911216 18023 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:36.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.923273 18023 webserver.cc:460] Webserver started at http://127.17.150.254:37771/ using document root <none> and password file <none>
13:46:36.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.924371 18023 fs_manager.cc:359] Metadata directory not provided
13:46:36.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.924624 18023 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.925228 18023 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:36.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.930193 18023 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data/instance:
13:46:36.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4cf4c963cf74daf9a8d2ff114de912d"
13:46:36.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:36 on dist-test-slave-v7p8"
13:46:36.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.931517 18023 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/wal/instance:
13:46:36.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4cf4c963cf74daf9a8d2ff114de912d"
13:46:36.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:36 on dist-test-slave-v7p8"
13:46:36.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.939165 18023 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
13:46:36.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.946455 18039 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:36.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.947543 18023 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
13:46:36.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.947882 18023 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/wal
13:46:36.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c4cf4c963cf74daf9a8d2ff114de912d"
13:46:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:36 on dist-test-slave-v7p8"
13:46:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.948274 18023 fs_report.cc:383] FS layout report
13:46:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/wal
13:46:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/wal
13:46:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data/data
13:46:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:36.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:36.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.988158 18023 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.989794 18023 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:36.990309 18023 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.081386 18023 rpc_server.cc:307] RPC server started. Bound to: 127.17.150.254:35555
13:46:37.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.081463 18090 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.150.254:35555 every 8 connection(s)
13:46:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.084146 18023 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/data/info.pb
13:46:37.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.089377 18011 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18023
13:46:37.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.089967 18011 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-0/wal/instance
13:46:37.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.090472 18091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.094846 18011 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/wal
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/logs
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data/info.pb
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:37.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.253:41841
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.253
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:37.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:37.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:37.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:37.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:37.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39703/jwks.json with env {}
13:46:37.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.107334 18091 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } has no permanent_uuid. Determining permanent_uuid...
13:46:37.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.140813 18091 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } has no permanent_uuid. Determining permanent_uuid...
13:46:37.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.143304 18042 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:37.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.149062 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.253:41841: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:37.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.198648 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } attempt: 1
13:46:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.203121 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.253:41841: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.271592 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } attempt: 2
13:46:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.275238 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.253:41841: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:37.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.366748 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } attempt: 3
13:46:37.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.370473 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.253:41841: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.425669 18093 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.426262 18093 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.426774 18093 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.457584 18093 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.457870 18093 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.458057 18093 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.458235 18093 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.458421 18093 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.458609 18093 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39703/jwks.json
13:46:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.493160 18093 master_runner.cc:386] Master server non-default flags:
13:46:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:37.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/wal
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.253:41841
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:37.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39703/jwks.json
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data/info.pb
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.253
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/logs
13:46:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:37.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:37.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.494473 18093 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.496188 18093 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:37.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.513840 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } attempt: 4
13:46:37.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.518178 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.253:41841: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:37.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.519001 18102 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:37.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:37.817687 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } attempt: 5
13:46:37.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.822543 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.253:41841: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.370030 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } attempt: 6
13:46:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:38.379594 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.253:41841: Network error: Client connection negotiation failed: client connection to 127.17.150.253:41841: connect: Connection refused (error 111)
13:46:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.519229 18103 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:37.519938 18105 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:38.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:38.653553 18104 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1130 milliseconds
13:46:38.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.653671 18093 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:46:38.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.655010 18093 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:38.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.657658 18093 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:38.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.659003 18093 hybrid_clock.cc:648] HybridClock initialized: now 1790603198658966 us; error 57 us; skew 500 ppm
13:46:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.659876 18093 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:38.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.667120 18093 webserver.cc:460] Webserver started at http://127.17.150.253:40739/ using document root <none> and password file <none>
13:46:38.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.668104 18093 fs_manager.cc:359] Metadata directory not provided
13:46:38.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.668311 18093 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:38.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.668846 18093 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.673547 18093 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data/instance:
13:46:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da871ce5c5474c73bebe99484bfbd2e6"
13:46:38.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:38 on dist-test-slave-v7p8"
13:46:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.674762 18093 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/wal/instance:
13:46:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da871ce5c5474c73bebe99484bfbd2e6"
13:46:38.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:38 on dist-test-slave-v7p8"
13:46:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.682348 18093 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
13:46:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.688607 18114 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:38.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.689659 18093 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
13:46:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.689988 18093 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/wal
13:46:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da871ce5c5474c73bebe99484bfbd2e6"
13:46:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:38 on dist-test-slave-v7p8"
13:46:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.690343 18093 fs_report.cc:383] FS layout report
13:46:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/wal
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/wal
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data/data
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.750100 18093 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.751549 18093 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:38.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.751964 18093 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.844913 18093 rpc_server.cc:307] RPC server started. Bound to: 127.17.150.253:41841
13:46:38.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.845037 18165 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.150.253:41841 every 8 connection(s)
13:46:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.847793 18093 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/data/info.pb
13:46:38.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.854710 18166 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:46:38.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.857455 18011 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18093
13:46:38.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.857784 18011 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-1/wal/instance
13:46:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.859892 18011 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/wal
13:46:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data
13:46:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:38.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/logs
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data/info.pb
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:38.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.252:46557
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.252
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:38.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39703/jwks.json with env {}
13:46:38.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.871119 18166 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } has no permanent_uuid. Determining permanent_uuid...
13:46:38.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.898809 18166 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } has no permanent_uuid. Determining permanent_uuid...
13:46:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.920197 18166 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } has no permanent_uuid. Determining permanent_uuid...
13:46:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:38.922767 18115 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:38.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:38.927364 18166 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:38.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:38.976886 18166 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 1
13:46:38.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:38.980468 18166 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.048836 18166 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 2
13:46:39.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.052450 18166 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.143872 18166 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 3
13:46:39.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.148922 18166 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.201239 18168 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.201788 18168 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.202239 18168 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:39.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.232236 18168 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:39.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.232538 18168 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.232829 18168 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.233091 18168 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.233294 18168 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:39.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.233433 18168 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39703/jwks.json
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.266656 18168 master_runner.cc:386] Master server non-default flags:
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/wal
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.252:46557
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39703/jwks.json
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data/info.pb
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.252
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:39.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/logs
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:39.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.267932 18168 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:39.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.270012 18168 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:39.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.289214 18178 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:39.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.292275 18166 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 4
13:46:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.295656 18166 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.440127 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } attempt: 7
13:46:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.452838 18091 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } has no permanent_uuid. Determining permanent_uuid...
13:46:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.458357 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.516858 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 1
13:46:39.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.521723 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.595042 18166 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 5
13:46:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.599213 18166 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.603109 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 2
13:46:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.606835 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.692209 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 3
13:46:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.696468 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:39.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.836859 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 4
13:46:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.841145 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:40.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.124572 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 5
13:46:40.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.127959 18091 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:40.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.146718 18166 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 6
13:46:40.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.155757 18166 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.150.252:46557: Network error: Client connection negotiation failed: client connection to 127.17.150.252:46557: connect: Connection refused (error 111)
13:46:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.290107 18179 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:40.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:39.291482 18168 server_base.cc:1034] running on GCE node
13:46:40.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:39.289372 18181 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.412714 18168 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.416451 18168 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:40.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.417881 18168 hybrid_clock.cc:648] HybridClock initialized: now 1790603200417821 us; error 78 us; skew 500 ppm
13:46:40.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.418746 18168 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.426966 18168 webserver.cc:460] Webserver started at http://127.17.150.252:40535/ using document root <none> and password file <none>
13:46:40.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.427923 18168 fs_manager.cc:359] Metadata directory not provided
13:46:40.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.428117 18168 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:40.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.428506 18168 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.433382 18168 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data/instance:
13:46:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119"
13:46:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:40 on dist-test-slave-v7p8"
13:46:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.434489 18168 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/wal/instance:
13:46:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119"
13:46:40.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:40 on dist-test-slave-v7p8"
13:46:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.442495 18168 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
13:46:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.449922 18193 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.451285 18168 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
13:46:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.451587 18168 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/wal
13:46:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119"
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:40 on dist-test-slave-v7p8"
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.451910 18168 fs_report.cc:383] FS layout report
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/wal
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/wal
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data/data
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.510675 18168 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.512794 18168 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:40.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.513199 18168 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:40.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.596812 18168 rpc_server.cc:307] RPC server started. Bound to: 127.17.150.252:46557
13:46:40.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.596906 18244 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.150.252:46557 every 8 connection(s)
13:46:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.599380 18168 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/data/info.pb
13:46:40.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.605302 18245 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:46:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.609760 18011 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18168
13:46:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.610128 18011 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/master-2/wal/instance
13:46:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.621209 18245 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } has no permanent_uuid. Determining permanent_uuid...
13:46:40.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.653909 18245 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } has no permanent_uuid. Determining permanent_uuid...
13:46:40.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.665164 18245 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } has no permanent_uuid. Determining permanent_uuid...
13:46:40.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.680701 18091 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 6
13:46:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.683207 18245 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119: Bootstrap starting.
13:46:40.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.690608 18245 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119: Neither blocks nor log segments found. Creating new log.
13:46:40.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.694124 18245 log.cc:824] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119: Log is configured to *not* fsync() on all Append() calls
13:46:40.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.703676 18091 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: Bootstrap starting.
13:46:40.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.704523 18245 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119: No bootstrap required, opened a new log
13:46:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.711037 18091 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: Neither blocks nor log segments found. Creating new log.
13:46:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.713757 18091 log.cc:824] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: Log is configured to *not* fsync() on all Append() calls
13:46:40.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.718631 18091 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: No bootstrap required, opened a new log
13:46:40.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.733911 18245 raft_consensus.cc:348] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:40.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.734737 18245 raft_consensus.cc:374] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:46:40.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.735015 18245 raft_consensus.cc:729] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa9ab3e7dafe41d8a341ac2c56eaf119, State: Initialized, Role: FOLLOWER
13:46:40.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.736196 18245 consensus_queue.cc:260] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.738894 18256 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.739604 18256 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: This master's current role is: FOLLOWER
13:46:40.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.740969 18245 sys_catalog.cc:564] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: configured and running, proceeding with master startup.
13:46:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.749111 18091 raft_consensus.cc:348] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.750029 18091 raft_consensus.cc:374] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:46:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.750324 18091 raft_consensus.cc:729] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c4cf4c963cf74daf9a8d2ff114de912d, State: Initialized, Role: FOLLOWER
13:46:40.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.751487 18091 consensus_queue.cc:260] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:40.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.755309 18261 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:40.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.755963 18261 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [sys.catalog]: This master's current role is: FOLLOWER
13:46:40.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.757315 18091 sys_catalog.cc:564] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [sys.catalog]: configured and running, proceeding with master startup.
13:46:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.771944 18199 tablet.cc:2369] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:46:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.787640 18272 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:46:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.788321 18272 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:46:40.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.776676 18168 master_runner.cc:186] Error getting master registration for 127.17.150.254:35555: OK, instance_id { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" instance_seqno: 1790603197058561 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:46:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.810849 18168 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:46:40.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.813833 18280 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:46:40.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.814401 18280 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:46:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.801110 18023 master_runner.cc:186] Error getting master registration for 127.17.150.253:41841: OK, instance_id { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" instance_seqno: 1790603198815825 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:46:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.835517 18023 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:46:40.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.862497 18256 raft_consensus.cc:482] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:46:40.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.863109 18256 raft_consensus.cc:504] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.869446 18066 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" is_pre_election: true
13:46:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.870452 18066 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa9ab3e7dafe41d8a341ac2c56eaf119 in term 0.
13:46:40.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.872643 18195 leader_election.cc:302] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aa9ab3e7dafe41d8a341ac2c56eaf119, c4cf4c963cf74daf9a8d2ff114de912d; no voters:
13:46:40.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.877533 18256 leader_election.cc:288] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c4cf4c963cf74daf9a8d2ff114de912d (127.17.150.254:35555), da871ce5c5474c73bebe99484bfbd2e6 (127.17.150.253:41841)
13:46:40.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.878875 18256 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Leader pre-election won for term 1
13:46:40.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.880029 18256 raft_consensus.cc:482] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
13:46:40.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.881199 18256 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 0 FOLLOWER]: Advancing to term 1
13:46:40.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.897186 18256 raft_consensus.cc:504] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:40.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.901404 18066 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c4cf4c963cf74daf9a8d2ff114de912d"
13:46:40.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.902012 18066 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [term 0 FOLLOWER]: Advancing to term 1
13:46:40.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.905709 18256 leader_election.cc:288] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [CANDIDATE]: Term 1 election: Requested vote from peers c4cf4c963cf74daf9a8d2ff114de912d (127.17.150.254:35555), da871ce5c5474c73bebe99484bfbd2e6 (127.17.150.253:41841)
13:46:40.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.909269 18066 raft_consensus.cc:2454] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa9ab3e7dafe41d8a341ac2c56eaf119 in term 1.
13:46:40.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.911164 18195 leader_election.cc:302] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: aa9ab3e7dafe41d8a341ac2c56eaf119, c4cf4c963cf74daf9a8d2ff114de912d; no voters:
13:46:40.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.912163 18256 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 1 FOLLOWER]: Leader election won for term 1
13:46:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.912792 18256 raft_consensus.cc:686] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [term 1 LEADER]: Becoming Leader. State: Replica: aa9ab3e7dafe41d8a341ac2c56eaf119, State: Running, Role: LEADER
13:46:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.914124 18256 consensus_queue.cc:237] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.924151 18282 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa9ab3e7dafe41d8a341ac2c56eaf119. Latest consensus state: current_term: 1 leader_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.925024 18282 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: This master's current role is: LEADER
13:46:40.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.932484 18284 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
13:46:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.940675 18284 catalog_manager.cc:1484] Initializing Kudu cluster ID...
13:46:40.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.960090 18066 raft_consensus.cc:1261] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [term 1 FOLLOWER]: Refusing update from remote peer aa9ab3e7dafe41d8a341ac2c56eaf119: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:46:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:40.961850 18256 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
13:46:41.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:40.999181 18045 tablet.cc:2369] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:46:41.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.012213 18261 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa9ab3e7dafe41d8a341ac2c56eaf119. Latest consensus state: current_term: 1 leader_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:41.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.013335 18261 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [sys.catalog]: This master's current role is: FOLLOWER
13:46:41.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.017221 18285 mvcc.cc:205] Tried to move back new op lower bound from 7334310711109836800 to 7334310710955307008. Current Snapshot: MvccSnapshot[applied={T|T < 7334310711109836800}]
13:46:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.019580 18286 mvcc.cc:205] Tried to move back new op lower bound from 7334310711109836800 to 7334310710955307008. Current Snapshot: MvccSnapshot[applied={T|T < 7334310711109836800}]
13:46:41.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.021317 18256 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:41.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.022082 18256 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: This master's current role is: LEADER
13:46:41.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.024972 18261 sys_catalog.cc:455] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:41.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.026086 18284 catalog_manager.cc:1347] Generated new cluster ID: dfb1cf7693184a65b07c36215c6f633a
13:46:41.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.026340 18284 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
13:46:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.035118 18261 sys_catalog.cc:458] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d [sys.catalog]: This master's current role is: FOLLOWER
13:46:41.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.066047 18284 catalog_manager.cc:1370] Generated new certificate authority record
13:46:41.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.069337 18284 catalog_manager.cc:1504] Loading token signing keys...
13:46:41.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.106284 18284 catalog_manager.cc:5868] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119: Generated new TSK 0
13:46:41.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.109733 18284 catalog_manager.cc:1514] Initializing in-progress tserver states...
13:46:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.135272 18011 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:41.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/wal
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/logs
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data/info.pb
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.193:0
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.150.193
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.193
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:41.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:46:41.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.216110 18166 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } attempt: 7
13:46:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.291273 18166 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: Bootstrap starting.
13:46:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.312737 18166 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: Neither blocks nor log segments found. Creating new log.
13:46:41.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.321586 18166 log.cc:824] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: Log is configured to *not* fsync() on all Append() calls
13:46:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.345351 18166 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: No bootstrap required, opened a new log
13:46:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.429170 18166 raft_consensus.cc:348] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:41.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.431761 18166 raft_consensus.cc:374] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:46:41.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.432827 18166 raft_consensus.cc:729] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: da871ce5c5474c73bebe99484bfbd2e6, State: Initialized, Role: FOLLOWER
13:46:41.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.435540 18166 consensus_queue.cc:260] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } }
13:46:41.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.460258 18141 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da871ce5c5474c73bebe99484bfbd2e6" is_pre_election: true
13:46:41.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.461903 18141 raft_consensus.cc:2454] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa9ab3e7dafe41d8a341ac2c56eaf119 in term 0.
13:46:41.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.465785 18140 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da871ce5c5474c73bebe99484bfbd2e6"
13:46:41.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.466881 18140 raft_consensus.cc:3037] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [term 0 FOLLOWER]: Advancing to term 1
13:46:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.476217 18166 sys_catalog.cc:564] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [sys.catalog]: configured and running, proceeding with master startup.
13:46:41.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.477393 18294 sys_catalog.cc:455] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:41.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.478971 18140 raft_consensus.cc:2454] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa9ab3e7dafe41d8a341ac2c56eaf119 in term 1.
13:46:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.483857 18139 raft_consensus.cc:1261] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [term 1 FOLLOWER]: Refusing update from remote peer aa9ab3e7dafe41d8a341ac2c56eaf119: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
13:46:41.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.490696 18294 sys_catalog.cc:458] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [sys.catalog]: This master's current role is: FOLLOWER
13:46:41.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.490584 18282 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [LEADER]: Connected to new peer: Peer: permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
13:46:41.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.509701 18120 tablet.cc:2369] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:46:41.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.543004 18298 mvcc.cc:205] Tried to move back new op lower bound from 7334310711109836800 to 7334310710955307008. Current Snapshot: MvccSnapshot[applied={T|T < 7334310711109836800}]
13:46:41.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.564075 18296 sys_catalog.cc:455] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:41.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.564252 18294 sys_catalog.cc:455] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa9ab3e7dafe41d8a341ac2c56eaf119. Latest consensus state: current_term: 1 leader_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:41.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.564946 18296 sys_catalog.cc:458] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [sys.catalog]: This master's current role is: FOLLOWER
13:46:41.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.565075 18294 sys_catalog.cc:458] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6 [sys.catalog]: This master's current role is: FOLLOWER
13:46:41.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.632421 18310 catalog_manager.cc:1259] Loaded cluster ID: dfb1cf7693184a65b07c36215c6f633a
13:46:41.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.632704 18310 catalog_manager.cc:1552] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: loading cluster ID for follower catalog manager: success
13:46:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.655468 18310 catalog_manager.cc:1574] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: acquiring CA information for follower catalog manager: success
13:46:41.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.661775 18310 catalog_manager.cc:1602] T 00000000000000000000000000000000 P da871ce5c5474c73bebe99484bfbd2e6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:46:41.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.677263 18290 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.677928 18290 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.678689 18290 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.737320 18290 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.738689 18290 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.150.193
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.801632 18290 tablet_server_runner.cc:78] Tablet server non-default flags:
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/wal
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.193:0
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data/info.pb
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.193
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:41.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.150.193
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/logs
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:41.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.803188 18290 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:41.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.805295 18290 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:41.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.820279 18280 catalog_manager.cc:1259] Loaded cluster ID: dfb1cf7693184a65b07c36215c6f633a
13:46:41.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.820923 18280 catalog_manager.cc:1552] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: loading cluster ID for follower catalog manager: success
13:46:41.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.836580 18280 catalog_manager.cc:1574] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: acquiring CA information for follower catalog manager: success
13:46:41.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.874286 18280 catalog_manager.cc:1602] T 00000000000000000000000000000000 P c4cf4c963cf74daf9a8d2ff114de912d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
13:46:41.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.901970 18318 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:41.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.987304 18282 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c4cf4c963cf74daf9a8d2ff114de912d" member_type: VOTER last_known_addr { host: "127.17.150.254" port: 35555 } } peers { permanent_uuid: "da871ce5c5474c73bebe99484bfbd2e6" member_type: VOTER last_known_addr { host: "127.17.150.253" port: 41841 } } peers { permanent_uuid: "aa9ab3e7dafe41d8a341ac2c56eaf119" member_type: VOTER last_known_addr { host: "127.17.150.252" port: 46557 } } }
13:46:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:41.990211 18282 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa9ab3e7dafe41d8a341ac2c56eaf119 [sys.catalog]: This master's current role is: LEADER
13:46:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.913779 18317 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:43.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:41.940271 18321 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:43.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:43.440465 18319 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:46:43.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.440514 18290 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:46:43.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.442399 18290 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:43.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.445175 18290 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:43.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.446604 18290 hybrid_clock.cc:648] HybridClock initialized: now 1790603203446628 us; error 170 us; skew 500 ppm
13:46:43.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.447810 18290 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:43.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.481835 18290 webserver.cc:460] Webserver started at http://127.17.150.193:34735/ using document root <none> and password file <none>
13:46:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.483703 18290 fs_manager.cc:359] Metadata directory not provided
13:46:43.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.484241 18290 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:43.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.485031 18290 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:43.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.492246 18290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data/instance:
13:46:43.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12444db07d96401bb32e3b64568f7ac0"
13:46:43.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:43 on dist-test-slave-v7p8"
13:46:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.494187 18290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/wal/instance:
13:46:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12444db07d96401bb32e3b64568f7ac0"
13:46:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:43 on dist-test-slave-v7p8"
13:46:43.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.519179 18290 fs_manager.cc:691] Time spent creating directory manager: real 0.024s user 0.008s sys 0.012s
13:46:43.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.536470 18331 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:43.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.537974 18290 fs_manager.cc:725] Time spent opening block manager: real 0.014s user 0.013s sys 0.001s
13:46:43.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.538373 18290 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/wal
13:46:43.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12444db07d96401bb32e3b64568f7ac0"
13:46:43.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:43 on dist-test-slave-v7p8"
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.538825 18290 fs_report.cc:383] FS layout report
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/wal
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/wal
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data/data
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:43.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.641006 18290 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:43.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.643741 18290 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:43.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.644333 18290 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:43.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.654886 18290 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:46:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.669504 18290 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:46:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.669808 18290 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:46:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.670116 18290 ts_tablet_manager.cc:610] Registered 0 tablets
13:46:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.670332 18290 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:46:43.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.901153 18290 rpc_server.cc:307] RPC server started. Bound to: 127.17.150.193:38775
13:46:43.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.901330 18443 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.150.193:38775 every 8 connection(s)
13:46:43.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.904311 18290 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/data/info.pb
13:46:43.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.912619 18011 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18290
13:46:43.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.913120 18011 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-0/wal/instance
13:46:43.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.952219 18011 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:43.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:43.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/wal
13:46:43.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/logs
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data/info.pb
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.194:0
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.150.194
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.194
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:43.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:43.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:46:44.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.997713 18444 heartbeater.cc:347] Connected to a master server at 127.17.150.252:46557
13:46:44.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.998195 18444 heartbeater.cc:464] Registering TS with master...
13:46:44.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:43.999406 18444 heartbeater.cc:511] Master 127.17.150.252:46557 requested a full tablet report, sending...
13:46:44.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.000850 18445 heartbeater.cc:347] Connected to a master server at 127.17.150.253:41841
13:46:44.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.001209 18445 heartbeater.cc:464] Registering TS with master...
13:46:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.002094 18445 heartbeater.cc:511] Master 127.17.150.253:41841 requested a full tablet report, sending...
13:46:44.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.004840 18131 ts_manager.cc:194] Registered new tserver with Master: 12444db07d96401bb32e3b64568f7ac0 (127.17.150.193:38775)
13:46:44.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.005672 18210 ts_manager.cc:194] Registered new tserver with Master: 12444db07d96401bb32e3b64568f7ac0 (127.17.150.193:38775)
13:46:44.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.008656 18210 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.150.193:46915
13:46:44.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.010123 18446 heartbeater.cc:347] Connected to a master server at 127.17.150.254:35555
13:46:44.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.010439 18446 heartbeater.cc:464] Registering TS with master...
13:46:44.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.011233 18446 heartbeater.cc:511] Master 127.17.150.254:35555 requested a full tablet report, sending...
13:46:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.014134 18056 ts_manager.cc:194] Registered new tserver with Master: 12444db07d96401bb32e3b64568f7ac0 (127.17.150.193:38775)
13:46:44.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.318095 18454 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:44.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.318609 18454 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:44.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.319033 18454 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:44.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.350114 18454 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:44.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.350909 18454 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.150.194
13:46:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.386404 18454 tablet_server_runner.cc:78] Tablet server non-default flags:
13:46:44.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/wal
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.194:0
13:46:44.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data/info.pb
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.194
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:44.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.150.194
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/logs
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.387588 18454 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.389289 18454 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:44.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.416451 18463 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:45.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.013392 18444 heartbeater.cc:503] Master 127.17.150.252:46557 was elected leader, sending a full tablet report...
13:46:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.417732 18461 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:44.422410 18454 server_base.cc:1034] running on GCE node
13:46:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:44.422305 18460 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:45.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.525494 18454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.527704 18454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:45.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.529059 18454 hybrid_clock.cc:648] HybridClock initialized: now 1790603205529027 us; error 37 us; skew 500 ppm
13:46:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.529883 18454 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.536033 18454 webserver.cc:460] Webserver started at http://127.17.150.194:41977/ using document root <none> and password file <none>
13:46:45.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.536981 18454 fs_manager.cc:359] Metadata directory not provided
13:46:45.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.537212 18454 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:45.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.537693 18454 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:45.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.542515 18454 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data/instance:
13:46:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc5cc1a6b4414a489bdeb48e0f89f761"
13:46:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:45 on dist-test-slave-v7p8"
13:46:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.543643 18454 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/wal/instance:
13:46:45.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc5cc1a6b4414a489bdeb48e0f89f761"
13:46:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:45 on dist-test-slave-v7p8"
13:46:45.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.552328 18454 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.005s
13:46:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.558550 18471 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:45.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.559587 18454 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
13:46:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.559918 18454 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/wal
13:46:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc5cc1a6b4414a489bdeb48e0f89f761"
13:46:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:45 on dist-test-slave-v7p8"
13:46:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.560267 18454 fs_report.cc:383] FS layout report
13:46:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/wal
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/wal
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data/data
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:45.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.596518 18454 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:45.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.598028 18454 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:45.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.598495 18454 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:45.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.601531 18454 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:46:45.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.605786 18454 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:46:45.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.606004 18454 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:46:45.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.606293 18454 ts_tablet_manager.cc:610] Registered 0 tablets
13:46:45.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.606452 18454 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:46:45.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.757001 18454 rpc_server.cc:307] RPC server started. Bound to: 127.17.150.194:37465
13:46:45.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.757100 18584 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.150.194:37465 every 8 connection(s)
13:46:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.759579 18454 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/data/info.pb
13:46:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.762830 18011 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18454
13:46:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.763417 18011 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-1/wal/instance
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.787838 18011 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/wal
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/logs
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data/info.pb
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.195:0
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.150.195
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.195
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
13:46:45.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.823856 18586 heartbeater.cc:347] Connected to a master server at 127.17.150.253:41841
13:46:45.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.824271 18586 heartbeater.cc:464] Registering TS with master...
13:46:45.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.825495 18586 heartbeater.cc:511] Master 127.17.150.253:41841 requested a full tablet report, sending...
13:46:45.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.826901 18585 heartbeater.cc:347] Connected to a master server at 127.17.150.252:46557
13:46:45.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.827231 18585 heartbeater.cc:464] Registering TS with master...
13:46:45.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.828759 18585 heartbeater.cc:511] Master 127.17.150.252:46557 requested a full tablet report, sending...
13:46:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.831377 18210 ts_manager.cc:194] Registered new tserver with Master: cc5cc1a6b4414a489bdeb48e0f89f761 (127.17.150.194:37465)
13:46:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.831568 18131 ts_manager.cc:194] Registered new tserver with Master: cc5cc1a6b4414a489bdeb48e0f89f761 (127.17.150.194:37465)
13:46:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.833436 18210 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.150.194:58139
13:46:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.848349 18587 heartbeater.cc:347] Connected to a master server at 127.17.150.254:35555
13:46:45.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.848821 18587 heartbeater.cc:464] Registering TS with master...
13:46:45.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.850133 18587 heartbeater.cc:511] Master 127.17.150.254:35555 requested a full tablet report, sending...
13:46:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:45.853164 18056 ts_manager.cc:194] Registered new tserver with Master: cc5cc1a6b4414a489bdeb48e0f89f761 (127.17.150.194:37465)
13:46:46.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.138278 18592 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:46.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.138842 18592 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.139793 18592 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:46.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.171041 18592 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:46.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.172046 18592 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.17.150.195
13:46:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:46.207229 18592 tablet_server_runner.cc:78] Tablet server non-default flags:
13:46:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:46.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.150.212:45849
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/wal
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.150.195:0
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data/info.pb
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.150.195
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.17.150.254:35555,127.17.150.253:41841,127.17.150.252:46557
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.17.150.195
13:46:46.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/logs
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:46.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:46.208793 18592 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:46.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:46.210603 18592 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:46.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.231899 18602 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:46.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:46.838428 18585 heartbeater.cc:503] Master 127.17.150.252:46557 was elected leader, sending a full tablet report...
13:46:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.232113 18601 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:47.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:46.232801 18604 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:47.347869 18603 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
13:46:47.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.348004 18592 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:46:47.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.351356 18592 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:47.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.353574 18592 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.354928 18592 hybrid_clock.cc:648] HybridClock initialized: now 1790603207354898 us; error 37 us; skew 500 ppm
13:46:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.355796 18592 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.361989 18592 webserver.cc:460] Webserver started at http://127.17.150.195:38637/ using document root <none> and password file <none>
13:46:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.362942 18592 fs_manager.cc:359] Metadata directory not provided
13:46:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.363147 18592 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.363617 18592 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:47.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.368317 18592 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data/instance:
13:46:47.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b591bc924dfe46038c6369c0f19dc3ef"
13:46:47.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:47 on dist-test-slave-v7p8"
13:46:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.369447 18592 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/wal/instance:
13:46:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b591bc924dfe46038c6369c0f19dc3ef"
13:46:47.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:47 on dist-test-slave-v7p8"
13:46:47.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.376890 18592 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
13:46:47.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.382673 18611 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:47.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.383661 18592 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
13:46:47.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.384002 18592 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/wal
13:46:47.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b591bc924dfe46038c6369c0f19dc3ef"
13:46:47.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:47 on dist-test-slave-v7p8"
13:46:47.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.384330 18592 fs_report.cc:383] FS layout report
13:46:47.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/wal
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/wal
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data/data
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:47.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.438445 18592 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.439960 18592 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:47.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.440414 18592 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:47.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.442988 18592 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
13:46:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.446990 18592 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
13:46:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.447182 18592 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
13:46:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.447471 18592 ts_tablet_manager.cc:610] Registered 0 tablets
13:46:47.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.447629 18592 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
13:46:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.602766 18592 rpc_server.cc:307] RPC server started. Bound to: 127.17.150.195:32813
13:46:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.602869 18724 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.150.195:32813 every 8 connection(s)
13:46:47.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.605355 18592 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/data/info.pb
13:46:47.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.613442 18011 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18592
13:46:47.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.613888 18011 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster2134410074234470799/ts-2/wal/instance
13:46:47.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.655169 18727 heartbeater.cc:347] Connected to a master server at 127.17.150.254:35555
13:46:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.655652 18727 heartbeater.cc:464] Registering TS with master...
13:46:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.656888 18726 heartbeater.cc:347] Connected to a master server at 127.17.150.253:41841
13:46:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.657028 18727 heartbeater.cc:511] Master 127.17.150.254:35555 requested a full tablet report, sending...
13:46:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.657232 18726 heartbeater.cc:464] Registering TS with master...
13:46:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.657995 18726 heartbeater.cc:511] Master 127.17.150.253:41841 requested a full tablet report, sending...
13:46:47.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.659659 18725 heartbeater.cc:347] Connected to a master server at 127.17.150.252:46557
13:46:47.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.659981 18725 heartbeater.cc:464] Registering TS with master...
13:46:47.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.659996 18056 ts_manager.cc:194] Registered new tserver with Master: b591bc924dfe46038c6369c0f19dc3ef (127.17.150.195:32813)
13:46:47.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.660772 18725 heartbeater.cc:511] Master 127.17.150.252:46557 requested a full tablet report, sending...
13:46:47.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.661334 18130 ts_manager.cc:194] Registered new tserver with Master: b591bc924dfe46038c6369c0f19dc3ef (127.17.150.195:32813)
13:46:47.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.662710 18210 ts_manager.cc:194] Registered new tserver with Master: b591bc924dfe46038c6369c0f19dc3ef (127.17.150.195:32813)
13:46:47.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.664516 18210 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.17.150.195:50407
13:46:47.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:47.671532 18011 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
13:46:47.682 [DEBUG - main] (MiniKuduCluster.java:195) Response:
13:46:47.683 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
13:46:47.693 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.150.254"
port: 35555
}
bound_http_address {
host: "127.17.150.254"
port: 37771
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.150.253"
port: 41841
}
bound_http_address {
host: "127.17.150.253"
port: 40739
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.150.252"
port: 46557
}
bound_http_address {
host: "127.17.150.252"
port: 40535
}
}
}
13:46:47.697 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:46:47.700 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.17.150.193"
port: 38775
}
bound_http_address {
host: "127.17.150.193"
port: 34735
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.17.150.194"
port: 37465
}
bound_http_address {
host: "127.17.150.194"
port: 41977
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.17.150.195"
port: 32813
}
bound_http_address {
host: "127.17.150.195"
port: 38637
}
}
}
13:46:47.700 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
13:46:47.937 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
13:46:48.122 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHFxKIK/test-tmp
13:46:48.123 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
13:46:48.123 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHFxKIK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
13:46:48.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
13:46:48.476 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:48.476 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
13:46:48.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:48.479691 18736 mini_oidc.cc:152] Starting JWKS server
13:46:48.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:48.483287 18736 webserver.cc:460] Webserver started at http://127.0.0.1:44567/ using document root <none> and password file <none>
13:46:48.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:48.483683 18736 mini_oidc.cc:189] Starting OIDC Discovery server
13:46:48.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:48.485410 18736 webserver.cc:460] Webserver started at http://127.0.0.1:33595/ using document root <none> and password file <none>
13:46:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
13:46:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/principal' for realm 'KRBTEST.COM',
13:46:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
13:46:48.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:48 dist-test-slave-v7p8 krb5kdc[18748](info): setting up network...
13:46:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
13:46:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:48 dist-test-slave-v7p8 krb5kdc[18748](info): set up 2 sockets
13:46:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:48 dist-test-slave-v7p8 krb5kdc[18748](info): commencing operation
13:46:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
13:46:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:48.667949 18725 heartbeater.cc:503] Master 127.17.150.252:46557 was elected leader, sending a full tablet report...
13:46:49.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:49.519234 18736 mini_kdc.cc:121] Time spent starting KDC: real 1.010s user 0.000s sys 0.028s
13:46:49.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
13:46:49.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:46:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
13:46:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
13:46:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:46:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
13:46:49.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
13:46:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:46:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
13:46:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
13:46:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/test-user.keytab.
13:46:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/test-user.keytab.
13:46:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:49 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:46:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:46:49.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T13:46:49Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
13:46:49.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T13:46:49Z Disabled control of system clock
13:46:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.76.62@KRBTEST.COM; defaulting to no policy
13:46:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:46:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.76.62@KRBTEST.COM" created.
13:46:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:46:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.76.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.62.keytab.
13:46:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.76.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.62.keytab.
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:49.697933 18736 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/logs
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/info.pb
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.62:33369
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.62
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.62.keytab
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.62
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:49.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:49.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:46:49.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:49.996263 18762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:49.996948 18762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:49.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:49.997648 18762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.029374 18762 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:46:50.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.029618 18762 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.030010 18762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.030351 18762 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:50.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.030670 18762 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.030995 18762 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:50.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.031200 18762 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44567/jwks.json
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:50.072721 18762 master_runner.cc:386] Master server non-default flags:
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.62.keytab
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.62
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.62:33369
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:46:50.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/info.pb
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.62
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/logs
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:50.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:50.074153 18762 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:50.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:50.075280 18762 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.097172 18769 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:51.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:50.098588 18768 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:51.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.450749 18770 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1352 milliseconds
13:46:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.452196 18771 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:51.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.454421 18762 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.359s user 0.506s sys 0.792s
13:46:51.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.454747 18762 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.359s user 0.506s sys 0.792s
13:46:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.454988 18762 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:46:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.456300 18762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.460214 18762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:51.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.461599 18762 hybrid_clock.cc:648] HybridClock initialized: now 1790603211461573 us; error 59 us; skew 500 ppm
13:46:51.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.462685 18762 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:51.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:51 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603211, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:46:51.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.470773 18762 init.cc:377] Logged in from keytab as kudu/127.18.76.62@KRBTEST.COM (short username kudu)
13:46:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.476760 18762 webserver.cc:460] Webserver started at http://127.18.76.62:34613/ using document root <none> and password file <none>
13:46:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.477738 18762 fs_manager.cc:359] Metadata directory not provided
13:46:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.477926 18762 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.478349 18762 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:51.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.483034 18762 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/instance:
13:46:51.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0058a5e62834508a89a48820f86a87f"
13:46:51.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:51 on dist-test-slave-v7p8"
13:46:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.484052 18762 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal/instance:
13:46:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0058a5e62834508a89a48820f86a87f"
13:46:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:51 on dist-test-slave-v7p8"
13:46:51.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.491724 18762 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
13:46:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.497494 18779 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.498708 18762 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.499029 18762 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0058a5e62834508a89a48820f86a87f"
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:51 on dist-test-slave-v7p8"
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.499333 18762 fs_report.cc:383] FS layout report
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/data
13:46:51.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:51.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.543947 18762 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.549804 18762 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:51.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.550236 18762 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.646308 18762 rpc_server.cc:307] RPC server started. Bound to: 127.18.76.62:33369
13:46:51.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.646406 18831 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.76.62:33369 every 8 connection(s)
13:46:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.649164 18762 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/info.pb
13:46:51.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.654249 18736 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18762
13:46:51.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.654778 18736 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal/instance
13:46:51.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.655421 18832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:46:51.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.670290 18832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } has no permanent_uuid. Determining permanent_uuid...
13:46:51.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.76.61@KRBTEST.COM; defaulting to no policy
13:46:51.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:46:51.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.76.61@KRBTEST.COM" created.
13:46:51.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:51 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603211, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.62@KRBTEST.COM for kudu/127.18.76.62@KRBTEST.COM
13:46:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:46:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.76.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.61.keytab.
13:46:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.76.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.61.keytab.
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.704913 18736 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/logs
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/info.pb
13:46:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.61:33787
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.61
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.61.keytab
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.61
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:51.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:51.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:46:51.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.712823 18832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } has no permanent_uuid. Determining permanent_uuid...
13:46:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.715114 18781 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.719082 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.61:33787: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:51.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.768530 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } attempt: 1
13:46:51.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.771972 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.61:33787: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.840319 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } attempt: 2
13:46:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.844434 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.61:33787: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:51.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:51.935853 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } attempt: 3
13:46:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.940553 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.61:33787: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:51.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.993078 18440 debug-util.cc:398] Leaking SignalData structure 0x7b0800041060 after lost signal to thread 18311
13:46:51.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:51.994503 18440 debug-util.cc:398] Leaking SignalData structure 0x7b08000bae40 after lost signal to thread 18443
13:46:52.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.036074 18838 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.036850 18838 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.037992 18838 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:52.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:52.083950 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } attempt: 4
13:46:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.088656 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.61:33787: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.089808 18838 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:46:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.090094 18838 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.090679 18838 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.091015 18838 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.091341 18838 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:52.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.091684 18838 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:52.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.091925 18838 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44567/jwks.json
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:52.138110 18838 master_runner.cc:386] Master server non-default flags:
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.61.keytab
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.61
13:46:52.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.61:33787
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/info.pb
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.61
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/logs
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:52.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:52.139552 18838 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:52.140957 18838 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:52.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.163856 18846 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:52.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:52.388098 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } attempt: 5
13:46:52.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.398098 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.61:33787: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:52.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:52.945694 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } attempt: 6
13:46:52.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.975034 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.61:33787: Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:46:53.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.163877 18848 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:53.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:52.163975 18845 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:53.695878 18847 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1533 milliseconds
13:46:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.695986 18838 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:46:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.700695 18838 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.706498 18838 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.708093 18838 hybrid_clock.cc:648] HybridClock initialized: now 1790603213708026 us; error 71 us; skew 500 ppm
13:46:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.709141 18838 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:53.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:53 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603213, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:46:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.723749 18838 init.cc:377] Logged in from keytab as kudu/127.18.76.61@KRBTEST.COM (short username kudu)
13:46:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.739871 18838 webserver.cc:460] Webserver started at http://127.18.76.61:41875/ using document root <none> and password file <none>
13:46:53.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.741039 18838 fs_manager.cc:359] Metadata directory not provided
13:46:53.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.741258 18838 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:53.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.741753 18838 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.747944 18838 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/instance:
13:46:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f39bf8407b246aaa1a4b2ddf85bec30"
13:46:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:53 on dist-test-slave-v7p8"
13:46:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.749702 18838 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal/instance:
13:46:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f39bf8407b246aaa1a4b2ddf85bec30"
13:46:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:53 on dist-test-slave-v7p8"
13:46:53.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.768790 18838 fs_manager.cc:691] Time spent creating directory manager: real 0.018s user 0.004s sys 0.011s
13:46:53.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.781272 18858 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:53.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.783187 18838 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.003s sys 0.005s
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.783610 18838 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f39bf8407b246aaa1a4b2ddf85bec30"
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:53 on dist-test-slave-v7p8"
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.784051 18838 fs_report.cc:383] FS layout report
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/data
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:53.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:53.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:53.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.850221 18838 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:53.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.859234 18838 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:53.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.860046 18838 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:53.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.980257 18838 rpc_server.cc:307] RPC server started. Bound to: 127.18.76.61:33787
13:46:53.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.980751 18909 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.76.61:33787 every 8 connection(s)
13:46:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.983698 18838 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/info.pb
13:46:53.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.985117 18736 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18838
13:46:53.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.985594 18736 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal/instance
13:46:53.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:53.995858 18910 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:46:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.18.76.60@KRBTEST.COM; defaulting to no policy
13:46:54.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.013458 18910 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } has no permanent_uuid. Determining permanent_uuid...
13:46:54.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.035511 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } attempt: 7
13:46:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:46:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.18.76.60@KRBTEST.COM" created.
13:46:54.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:54 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603213, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.61@KRBTEST.COM for kudu/127.18.76.62@KRBTEST.COM
13:46:54.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:54 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603211, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.62@KRBTEST.COM for kudu/127.18.76.61@KRBTEST.COM
13:46:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.164412 18910 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } has no permanent_uuid. Determining permanent_uuid...
13:46:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.166935 18832 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } has no permanent_uuid. Determining permanent_uuid...
13:46:54.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.175293 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.233709 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 1
13:46:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.237215 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.318572 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 2
13:46:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.322763 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.408113 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 3
13:46:54.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.411482 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:54 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603213, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.61@KRBTEST.COM for kudu/127.18.76.61@KRBTEST.COM
13:46:54.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.442369 18914 sasl_common.cc:323] Time spent calling sasl_client_start(): real 0.266s user 0.001s sys 0.000s
13:46:54.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.451701 18910 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } has no permanent_uuid. Determining permanent_uuid...
13:46:54.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.453807 18862 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.457273 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.506717 18910 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 1
13:46:54.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.510171 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.551849 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 4
13:46:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.555162 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.578511 18910 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 2
13:46:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.581904 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.673225 18910 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 3
13:46:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.676443 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.819900 18910 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 4
13:46:54.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.823278 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:54.838524 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 5
13:46:54.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:54.842024 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:55.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:55.122668 18910 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 5
13:46:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.126935 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
13:46:55.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.76.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.60.keytab.
13:46:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.18.76.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.60.keytab.
13:46:55.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.135330 18736 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.18.76.60: real 1.148s user 0.003s sys 0.017s
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:55.137075 18736 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/logs
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/info.pb
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.60:34005
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.60
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.60.keytab
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.60
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:46:55.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:55.394431 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 6
13:46:55.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.400951 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.517884 18922 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.518523 18922 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:55.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.519245 18922 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:55.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.552537 18922 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:46:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.552822 18922 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.553239 18922 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:55.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.553545 18922 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.553826 18922 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.554131 18922 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.554332 18922 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44567/jwks.json
13:46:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:55.592303 18922 master_runner.cc:386] Master server non-default flags:
13:46:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:55.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.60.keytab
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.60
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.60:34005
13:46:55.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/info.pb
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.60
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/logs
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:55.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:55.593743 18922 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:55.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:55.595075 18922 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:55.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.614940 18931 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:55.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:55.674288 18910 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 6
13:46:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.682097 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.434331 18832 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 7
13:46:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:56.443547 18832 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.742591 18910 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } attempt: 7
13:46:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:56.760696 18910 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.18.76.60:34005: Network error: Client connection negotiation failed: client connection to 127.18.76.60:34005: connect: Connection refused (error 111)
13:46:56.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.620806 18934 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:56.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:55.617797 18932 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:56.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:56.802507 18933 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1181 milliseconds
13:46:56.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.802621 18922 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
13:46:56.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.803845 18922 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.806445 18922 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:56.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.807824 18922 hybrid_clock.cc:648] HybridClock initialized: now 1790603216807775 us; error 58 us; skew 500 ppm
13:46:56.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.808877 18922 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:56.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:56 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603216, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:46:56.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.815500 18922 init.cc:377] Logged in from keytab as kudu/127.18.76.60@KRBTEST.COM (short username kudu)
13:46:56.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.820680 18922 webserver.cc:460] Webserver started at http://127.18.76.60:43765/ using document root <none> and password file <none>
13:46:56.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.821655 18922 fs_manager.cc:359] Metadata directory not provided
13:46:56.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.821833 18922 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:56.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.822206 18922 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
13:46:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.827816 18922 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/instance:
13:46:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6487a53ff0f4d8dbc25c453848464db"
13:46:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:56 on dist-test-slave-v7p8"
13:46:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.829129 18922 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal/instance:
13:46:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6487a53ff0f4d8dbc25c453848464db"
13:46:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:56 on dist-test-slave-v7p8"
13:46:56.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.836397 18922 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
13:46:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.841912 18950 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:56.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.843127 18922 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
13:46:56.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.843508 18922 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:46:56.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6487a53ff0f4d8dbc25c453848464db"
13:46:56.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:56 on dist-test-slave-v7p8"
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.843928 18922 fs_report.cc:383] FS layout report
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/data
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:56.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.895344 18922 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.901221 18922 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.901682 18922 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:56.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.992983 18922 rpc_server.cc:307] RPC server started. Bound to: 127.18.76.60:34005
13:46:56.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.993070 19001 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.76.60:34005 every 8 connection(s)
13:46:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.995524 18922 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/info.pb
13:46:56.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.998469 18736 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 18922
13:46:56.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:56.998951 18736 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal/instance
13:46:57.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.002064 19002 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
13:46:57.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.030711 19002 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } has no permanent_uuid. Determining permanent_uuid...
13:46:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.76.60@KRBTEST.COM
13:46:57.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603216, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.60@KRBTEST.COM for kudu/127.18.76.62@KRBTEST.COM
13:46:57.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.073868 19002 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } has no permanent_uuid. Determining permanent_uuid...
13:46:57.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.76.61@KRBTEST.COM
13:46:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603216, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.60@KRBTEST.COM for kudu/127.18.76.61@KRBTEST.COM
13:46:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.112481 19002 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } has no permanent_uuid. Determining permanent_uuid...
13:46:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603209, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.76.62@KRBTEST.COM
13:46:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603216, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.60@KRBTEST.COM for kudu/127.18.76.60@KRBTEST.COM
13:46:57.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.145638 18736 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
13:46:57.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.146857 19002 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Bootstrap starting.
13:46:57.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.154443 19002 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Neither blocks nor log segments found. Creating new log.
13:46:57.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.156800 19002 log.cc:824] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Log is configured to *not* fsync() on all Append() calls
13:46:57.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
13:46:57.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.161697 19002 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: No bootstrap required, opened a new log
13:46:57.168 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
13:46:57.168 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
13:46:57.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.18.76.62"
port: 33369
}
bound_http_address {
host: "127.18.76.62"
port: 34613
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.18.76.61"
port: 33787
}
bound_http_address {
host: "127.18.76.61"
port: 41875
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.18.76.60"
port: 34005
}
bound_http_address {
host: "127.18.76.60"
port: 43765
}
}
}
13:46:57.172 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
13:46:57.172 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:57.172 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:46:57.173 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:46:57.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.190467 19002 raft_consensus.cc:348] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:46:57.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.191293 19002 raft_consensus.cc:374] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:46:57.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.191601 19002 raft_consensus.cc:729] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a6487a53ff0f4d8dbc25c453848464db, State: Initialized, Role: FOLLOWER
13:46:57.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.192473 19002 consensus_queue.cc:260] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:46:57.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.195390 19012 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } } }
13:46:57.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.196298 19012 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [sys.catalog]: This master's current role is: FOLLOWER
13:46:57.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.196666 19002 sys_catalog.cc:564] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [sys.catalog]: configured and running, proceeding with master startup.
13:46:57.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603217, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:46:57.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:46:57.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:57.233 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc
13:46:57.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.250706 19024 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:46:57.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.251048 19024 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:46:57.250 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:46:57.252 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc
13:46:57.254 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:46:57.257 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
13:46:57.259 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
13:46:57.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
13:46:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.239487 18922 master_runner.cc:186] Error getting master registration for 127.18.76.62:33369: OK, instance_id { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" instance_seqno: 1790603211619687 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
13:46:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.278748 18922 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:46:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:57 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603217, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:46:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
13:46:57.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:57.335 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc
13:46:57.337 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
13:46:57.337 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.76.61:33787
13:46:57.339 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
13:46:57.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.340221 18736 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18838
13:46:57.358 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:57.358 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.76.62:33369
13:46:57.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
13:46:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.359505 18736 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18762
13:46:57.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:57.384 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.18.76.60:34005
13:46:57.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
13:46:57.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.385382 18736 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18922
13:46:57.405 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:57.405 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.76.61:33787
13:46:57.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.409334 18736 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:57.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/logs
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/info.pb
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.61:33787
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.61
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41875
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.61.keytab
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.61
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:57.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:46:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.706470 19028 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.707116 19028 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.707813 19028 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.739151 19028 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:46:57.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.739409 19028 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.739789 19028 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.740093 19028 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.740374 19028 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:57.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.740665 19028 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:57.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.740880 19028 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44567/jwks.json
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.778635 19028 master_runner.cc:386] Master server non-default flags:
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.61.keytab
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.61
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.61:33787
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:46:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/info.pb
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.61
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41875
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/logs
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.780140 19028 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:57.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.781419 19028 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:57.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.798969 19036 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.798975 19035 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:57.799580 19038 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:46:58.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:57.801270 19028 server_base.cc:1034] running on GCE node
13:46:58.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.912170 19028 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:46:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.914772 19028 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:46:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.916147 19028 hybrid_clock.cc:648] HybridClock initialized: now 1790603218916107 us; error 42 us; skew 500 ppm
13:46:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.917140 19028 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:46:58.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:46:58 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603218, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:46:58.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.926222 19028 init.cc:377] Logged in from keytab as kudu/127.18.76.61@KRBTEST.COM (short username kudu)
13:46:58.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.930725 19028 webserver.cc:460] Webserver started at http://127.18.76.61:41875/ using document root <none> and password file <none>
13:46:58.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.931651 19028 fs_manager.cc:359] Metadata directory not provided
13:46:58.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.931864 19028 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:46:58.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.939586 19028 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
13:46:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.943926 19047 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:46:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.945101 19028 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
13:46:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.945420 19028 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f39bf8407b246aaa1a4b2ddf85bec30"
13:46:58.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:53 on dist-test-slave-v7p8"
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.947674 19028 fs_report.cc:383] FS layout report
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/data
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:46:58.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:46:58.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.993000 19028 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:46:58.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.998775 19028 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:46:58.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:58.999287 19028 kserver.cc:163] Server-wide thread pool size limit: 3276
13:46:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.068068 19028 rpc_server.cc:307] RPC server started. Bound to: 127.18.76.61:33787
13:46:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.068209 19098 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.76.61:33787 every 8 connection(s)
13:46:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.070863 19028 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/data/info.pb
13:46:59.078 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:46:59.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.077169 18736 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 19028
13:46:59.078 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.76.62:33369
13:46:59.078 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
13:46:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.080219 18736 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:46:59.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/logs
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/info.pb
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:59.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.62:33369
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.62
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34613
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.62.keytab
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.62
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:59.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:46:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.083382 19099 sys_catalog.cc:263] Verifying existing consensus state
13:46:59.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 13:46:59.084275 19099 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:46:59.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.084792 19028 master.cc:537] Master@127.18.76.61:33787 shutting down...
13:46:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.143414 19028 master.cc:552] Master@127.18.76.61:33787 shutdown complete.
13:46:59.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:46:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.407723 19101 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:46:59.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.408381 19101 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:46:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.409272 19101 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:46:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.440165 19101 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:46:59.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.440425 19101 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:46:59.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.440841 19101 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:46:59.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.441136 19101 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:46:59.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.441454 19101 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:46:59.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.441741 19101 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:46:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.441983 19101 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44567/jwks.json
13:46:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.478109 19101 master_runner.cc:386] Master server non-default flags:
13:46:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:46:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:46:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.62.keytab
13:46:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.62
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.62:33369
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/info.pb
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.62
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34613
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/logs
13:46:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.479547 19101 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:46:59.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.480939 19101 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:46:59.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.497138 19108 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.497200 19111 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:46:59.499257 19101 server_base.cc:1034] running on GCE node
13:47:00.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:46:59.497164 19109 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:00.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.623746 19101 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.626523 19101 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.627880 19101 hybrid_clock.cc:648] HybridClock initialized: now 1790603220627842 us; error 46 us; skew 500 ppm
13:47:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.628692 19101 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:47:00 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603220, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:00.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.634588 19101 init.cc:377] Logged in from keytab as kudu/127.18.76.62@KRBTEST.COM (short username kudu)
13:47:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.639673 19101 webserver.cc:460] Webserver started at http://127.18.76.62:34613/ using document root <none> and password file <none>
13:47:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.640527 19101 fs_manager.cc:359] Metadata directory not provided
13:47:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.640777 19101 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:00.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.649168 19101 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
13:47:00.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.653949 19119 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:00.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.655090 19101 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
13:47:00.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.655380 19101 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:47:00.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e0058a5e62834508a89a48820f86a87f"
13:47:00.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:51 on dist-test-slave-v7p8"
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.657737 19101 fs_report.cc:383] FS layout report
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/data
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:00.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.691903 19101 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:47:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.697594 19101 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:00.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.697999 19101 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.769757 19101 rpc_server.cc:307] RPC server started. Bound to: 127.18.76.62:33369
13:47:00.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.769840 19170 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.76.62:33369 every 8 connection(s)
13:47:00.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.772787 19101 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/data/info.pb
13:47:00.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.778892 18736 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 19101
13:47:00.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:00.780 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.18.76.60:34005
13:47:00.780 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.782078 18736 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/logs
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/info.pb
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
13:47:00.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.60:34005
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.60
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.60.keytab
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.60
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:47:00.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:47:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:47:00.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
13:47:00.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.785288 19171 sys_catalog.cc:263] Verifying existing consensus state
13:47:00.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 13:47:00.786234 19171 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:47:00.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.786717 19101 master.cc:537] Master@127.18.76.62:33369 shutting down...
13:47:00.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:00.858011 19101 master.cc:552] Master@127.18.76.62:33369 shutdown complete.
13:47:00.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
13:47:01.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.128897 19173 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
13:47:01.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.129456 19173 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
13:47:01.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.130080 19173 flags.cc:425] Enabled unsafe flag: --never_fsync=true
13:47:01.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.161342 19173 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
13:47:01.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.161585 19173 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
13:47:01.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.161839 19173 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
13:47:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.162051 19173 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
13:47:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.162333 19173 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
13:47:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.162606 19173 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
13:47:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.162767 19173 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44567/jwks.json
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:01.200115 19173 master_runner.cc:386] Master server non-default flags:
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.18.76.20:42199
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.18.76.62:33369,127.18.76.61:33787,127.18.76.60:34005
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/krb5kdc/kudu_127.18.76.60.keytab
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.18.76.60
13:47:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.18.76.60:34005
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44567/jwks.json
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/info.pb
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.18.76.60
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43765
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
13:47:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/logs
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 12:46:33 UTC on ccycloud.quasar-ezcsfi.root.comops.site
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677205
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:01.201367 19173 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
13:47:01.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:01.202440 19173 file_cache.cc:493] Constructed file cache file cache with capacity 419430
13:47:01.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.219929 19182 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:01.221699 19173 server_base.cc:1034] running on GCE node
13:47:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.219997 19179 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:02.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:01.219985 19180 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
13:47:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.349722 19173 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
13:47:02.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.352728 19173 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
13:47:02.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.354156 19173 hybrid_clock.cc:648] HybridClock initialized: now 1790603222354126 us; error 51 us; skew 500 ppm
13:47:02.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.354998 19173 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
13:47:02.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:47:02 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603222, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:02.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.361196 19173 init.cc:377] Logged in from keytab as kudu/127.18.76.60@KRBTEST.COM (short username kudu)
13:47:02.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.366504 19173 webserver.cc:460] Webserver started at http://127.18.76.60:43765/ using document root <none> and password file <none>
13:47:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.367421 19173 fs_manager.cc:359] Metadata directory not provided
13:47:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.367640 19173 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
13:47:02.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.375556 19173 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
13:47:02.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.380043 19191 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
13:47:02.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.381265 19173 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
13:47:02.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.381548 19173 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data,/tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:47:02.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6487a53ff0f4d8dbc25c453848464db"
13:47:02.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 13:46:56 on dist-test-slave-v7p8"
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.383863 19173 fs_report.cc:383] FS layout report
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/wal
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/data
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
13:47:02.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
13:47:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.415949 19173 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
13:47:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.421105 19173 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
13:47:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.421490 19173 kserver.cc:163] Server-wide thread pool size limit: 3276
13:47:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.487957 19173 rpc_server.cc:307] RPC server started. Bound to: 127.18.76.60:34005
13:47:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.488070 19242 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.18.76.60:34005 every 8 connection(s)
13:47:02.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.490731 19173 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHFxKIK/test-tmp/mini-kudu-cluster5029776268988300800/master-2/data/info.pb
13:47:02.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.497493 18736 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu as pid 19173
13:47:02.498 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
13:47:02.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.502018 19243 sys_catalog.cc:263] Verifying existing consensus state
13:47:02.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.507436 19243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Bootstrap starting.
13:47:02.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.539580 19243 log.cc:824] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Log is configured to *not* fsync() on all Append() calls
13:47:02.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.544471 19243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
13:47:02.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.545151 19243 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Bootstrap complete.
13:47:02.550 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.572011 19243 raft_consensus.cc:348] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.575542 19243 raft_consensus.cc:374] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
13:47:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.575853 19243 raft_consensus.cc:729] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a6487a53ff0f4d8dbc25c453848464db, State: Initialized, Role: FOLLOWER
13:47:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.576887 19243 consensus_queue.cc:260] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.585754 19247 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } } }
13:47:02.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.587644 19243 sys_catalog.cc:564] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [sys.catalog]: configured and running, proceeding with master startup.
13:47:02.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.587775 19247 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [sys.catalog]: This master's current role is: FOLLOWER
13:47:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:02.647086 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:02.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:02.647450 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:02.651 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:02.652 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.18.76.60:34005(127.18.76.60:34005)] connecting to peer
13:47:02.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.659497 19247 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.660036 19247 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:02.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.663657 19247 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:02.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:02.665848 19194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:02.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:02.676333 19197 tablet.cc:2369] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: Can't schedule compaction. Clean time has not been advanced past its initial value.
13:47:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.640235 19173 master_runner.cc:186] Error getting master registration for 127.18.76.62:33369: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111),
13:47:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.678684 19173 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:02.681586 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:02.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:02.686443 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:02.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.686877 19193 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:02.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:02.687690 19247 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:02.720 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:02.721 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:02.722 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:02.723 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:02.724 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:02.726 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:02.727 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.18.76.60:34005(127.18.76.60:34005)] Successfully connected to peer
13:47:02.810 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
13:47:02.836 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.18.76.60
13:47:03.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:47:03 dist-test-slave-v7p8 krb5kdc[18748](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603217, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.18.76.60@KRBTEST.COM
13:47:03.204 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xd59f79f2, L:/127.0.0.1:49784 - R:/127.18.76.60:34005] using SASL/GSSAPI
13:47:03.220 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.225 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.227 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.230 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.240 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.243 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.244 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.246 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.247 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.254 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.263 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.264 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.270 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.274 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.304 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.310 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.311 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.311 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.313 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.344 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.345 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.346 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.346 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.346 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.352 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.364 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.367 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.368 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.368 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.371 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.371 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.372 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.373 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.484 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.486 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.487 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.493 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.494 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.494 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:03.650641 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:03.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:03.650897 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:03.724 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.725 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.727 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.727 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.730 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.731 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.731 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.731 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:03.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:03.925 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:03.926 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.926 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.928 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:03.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:03.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:04.656070 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:04.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:04.656394 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:04.898084 19268 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:04.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:04.898478 19268 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:04.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:04.900108 19268 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:04.906602 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:04.907363 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:04.907734 19193 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:04.908360 19268 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:04.964 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:04.965 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:04.967 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:04.967 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:04.967 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:04.968 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:04.968 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:04.968 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:04.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:05.659350 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:05.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:05.659623 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:05.683660 18581 debug-util.cc:398] Leaking SignalData structure 0x7b0800042060 after lost signal to thread 18455
13:47:05.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:05.684491 18581 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc560 after lost signal to thread 18584
13:47:06.404 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:06.405 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:06.406 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:06.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:06.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:06.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:06.412 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:06.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:06.412 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:06.441839 19276 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:06.442232 19276 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:06.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:06.443774 19276 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:06.448652 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:06.449278 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:06.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:06.449576 19193 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:06.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:06.450150 19276 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:06.462954 18162 debug-util.cc:398] Leaking SignalData structure 0x7b08000aec00 after lost signal to thread 18097
13:47:06.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:06.463686 18162 debug-util.cc:398] Leaking SignalData structure 0x7b080005c0e0 after lost signal to thread 18165
13:47:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:06.663760 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:06.664119 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:07.666916 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:07.667171 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:07.681676 19173 master_runner.cc:186] Error getting master registration for 127.18.76.62:33369: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111),
13:47:07.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:07.682137 19173 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:08.598023 19283 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:08.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:08.598372 19283 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:08.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:08.599737 19283 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:08.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:08.602039 19194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111) [suppressed 5 similar messages]
13:47:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:08.620132 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:08.620603 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:08.621006 19193 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:08.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:08.621730 19283 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:08.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:08.669966 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:08.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:08.670243 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:09.404 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:09.405 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:09.406 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:09.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:09.407 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:09.407 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:09.408 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:09.408 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:09.409 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:09.673187 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:09.673460 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:09.704 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:09.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:09.706 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:09.707 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:09.707 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:09.708 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:09.708 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:09.708 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:09.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:10.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:10.676165 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:10.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:10.676422 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:11.644 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:11.645 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:11.646 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:11.646 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:11.646 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:11.647 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:11.647 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:11.648 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:11.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:11.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:11.679849 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:11.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:11.680166 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:12.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:47:12 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603232, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:12.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:12.365015 19187 init.cc:295] Successfully reacquired a new kerberos TGT
13:47:12.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:12.683058 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:12.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:12.683313 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:12.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:12.685356 19173 master_runner.cc:186] Error getting master registration for 127.18.76.62:33369: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111),
13:47:12.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:12.685768 19173 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:13.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:13.686034 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:13.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:13.686278 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:14.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:14.172114 18241 debug-util.cc:398] Leaking SignalData structure 0x7b08000bab20 after lost signal to thread 18173
13:47:14.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:14.172734 18241 debug-util.cc:398] Leaking SignalData structure 0x7b08000a21a0 after lost signal to thread 18244
13:47:14.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:14.689157 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:14.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:14.689430 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:15.124 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:15.126 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:15.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:15.128 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:15.128 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:15.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:15.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:15.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:15.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:15.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:15.692162 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:15.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:15.692415 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:15.998775 19294 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:15.999251 19294 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:16.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:16.000983 19294 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:16.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:16.003005 19194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111) [suppressed 1 similar messages]
13:47:16.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:16.005452 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:16.005849 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:16.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:16.006158 19194 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:16.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:16.006762 19294 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:16.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:16.695212 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:16.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:16.695463 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:17.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:17.688405 19173 master_runner.cc:186] Error getting master registration for 127.18.76.62:33369: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111),
13:47:17.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:17.688858 19173 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:17.698364 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:17.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:17.698719 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:17.824976 19301 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:17.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:17.825399 19301 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:17.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:17.826934 19301 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:17.830839 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:17.831239 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:17.831571 19193 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:17.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:17.832090 19301 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:18.701864 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:18.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:18.702131 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:19.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:19.005 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:19.006 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:19.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:19.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:19.008 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:19.008 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:19.008 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:19.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:19.577435 19307 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:19.577874 19307 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:19.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:19.579407 19307 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:19.583912 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:19.584343 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:19.584682 19193 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:19.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:19.585217 19307 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:19.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:19.704982 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:19.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:19.705224 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:19.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:19.925 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:19.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:19.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:19.927 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:19.927 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:19.928 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:19.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:19.929 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:20.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:20.708096 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:20.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:20.708348 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:21.711910 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:21.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:21.712280 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:22.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 13:47:22 dist-test-slave-v7p8 krb5kdc[18748](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790603242, etypes {rep=17 tkt=17 ses=17}, kudu/127.18.76.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
13:47:22.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:22.367317 19187 init.cc:295] Successfully reacquired a new kerberos TGT
13:47:22.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:22.545 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:22.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:22.546 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:22.547 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:22.548 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:22.548 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:22.548 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:22.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:22.691924 19173 master_runner.cc:186] Error getting master registration for 127.18.76.62:33369: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111),
13:47:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:22.692384 19173 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:22.715137 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:22.715394 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:23.718052 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:23.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:23.718303 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:24.721028 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:24.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:24.721266 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:25.243 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:25.245 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:25.246 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:25.246 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:25.246 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:25.247 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:25.247 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:25.247 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:25.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:25.723930 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:25.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:25.724172 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:26.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:26.726915 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:26.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:26.727180 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:27.694921 19173 master_runner.cc:186] Error getting master registration for 127.18.76.62:33369: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111),
13:47:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:27.695252 19173 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
13:47:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:27.729986 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:27.730245 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:27.923 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:27.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:27.925 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:27.925 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:27.925 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:27.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:27.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:27.926 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:27.928 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:28.732976 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:28.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:28.733214 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:29.164 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:29.165 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:29.165 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:29.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:29.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:29.167 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:29.167 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:29.167 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:29.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:29.724 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:29.725 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:29.725 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:29.726 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:29.726 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:29.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:29.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:29.727 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:29.728 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:29.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:29.736025 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:29.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:29.736266 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:30.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:30.739097 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:30.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:30.739370 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:31.323 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:31.324 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:31.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:31.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:31.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:31.326 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:31.326 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:31.326 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:31.329 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:31.587128 19325 raft_consensus.cc:482] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
13:47:31.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:31.587738 19325 raft_consensus.cc:504] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0058a5e62834508a89a48820f86a87f" member_type: VOTER last_known_addr { host: "127.18.76.62" port: 33369 } } peers { permanent_uuid: "9f39bf8407b246aaa1a4b2ddf85bec30" member_type: VOTER last_known_addr { host: "127.18.76.61" port: 33787 } } peers { permanent_uuid: "a6487a53ff0f4d8dbc25c453848464db" member_type: VOTER last_known_addr { host: "127.18.76.60" port: 34005 } }
13:47:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:31.589718 19325 leader_election.cc:288] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369), 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787)
13:47:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:31.592010 19194 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111) [suppressed 5 similar messages]
13:47:31.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:31.595117 19193 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9f39bf8407b246aaa1a4b2ddf85bec30 (127.18.76.61:33787): Network error: Client connection negotiation failed: client connection to 127.18.76.61:33787: connect: Connection refused (error 111)
13:47:31.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:31.595510 19194 leader_election.cc:334] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e0058a5e62834508a89a48820f86a87f (127.18.76.62:33369): Network error: Client connection negotiation failed: client connection to 127.18.76.62:33369: connect: Connection refused (error 111)
13:47:31.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:31.595808 19194 leader_election.cc:302] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a6487a53ff0f4d8dbc25c453848464db; no voters: 9f39bf8407b246aaa1a4b2ddf85bec30, e0058a5e62834508a89a48820f86a87f
13:47:31.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:31.596354 19325 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
13:47:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:31.742697 19258 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a6487a53ff0f4d8dbc25c453848464db: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
13:47:31.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 13:47:31.743000 19258 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
13:47:32.043 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.61:33787(127.18.76.61:33787)] connecting to peer
13:47:32.044 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.18.76.62:33369(127.18.76.62:33369)] connecting to peer
13:47:32.045 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:32.045 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.18.76.61:33787(127.18.76.61:33787)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:32.045 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.61:33787: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787
13:47:32.046 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:32.046 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.62:33369(127.18.76.62:33369)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:32.046 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.18.76.62:33369: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
13:47:32.048 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005; will retry
13:47:32.053 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29546), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [215ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [219ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [710ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [718ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [724ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [733ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [734ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [745ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [746ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [748ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [760ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [764ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [767ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [802ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [805ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [807ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [840ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [843ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [845ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [858ms] refreshing cache from master, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [862ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [865ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [867ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [980ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [981ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [986ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [987ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [988ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1217ms] refreshing cache from master, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1225ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1227ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1228ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1425ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1426ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3897ms] refreshing cache from master, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [3899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [3905ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [3906ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3906ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [6898ms] refreshing cache from master, [6898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [6899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [6900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [6900ms] too many traces: truncated at 100 traces, deferred=Deferred@1335312408(state=PENDING, result=null, callback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
13:47:32.058 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.18.76.60:34005(127.18.76.60:34005)] disconnecting while in state READY
13:47:32.059 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.18.76.62:33369(127.18.76.62:33369)] disconnecting while in state TERMINATED
13:47:32.059 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.18.76.61:33787(127.18.76.61:33787)] disconnecting while in state TERMINATED
13:47:32.064 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:47:32.065 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:47:32.067 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.18.76.60:34005(127.18.76.60:34005)] handling channelInactive
13:47:32.067 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.18.76.60:34005(127.18.76.60:34005)] cleaning up while in state READY due to: connection closed
13:47:32.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:32.070976 18736 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 19173
13:47:33.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T13:47:33Z chronyd exiting
13:47:34.212 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
13:47:34.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:34.213985 18011 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18290
13:47:34.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:34.235850 18011 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18454
13:47:34.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:34.261228 18011 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18592
13:47:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:34.283458 18011 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18023
13:47:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:34.311554 18011 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18093
13:47:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 13:47:34.339304 18011 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHFxKIK/build/tsan/bin/kudu with pid 18168
13:47:35.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T13:47:35Z chronyd exiting
13:47:36.432 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29546), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [215ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [219ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [710ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [718ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [724ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [733ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [734ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [745ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [746ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [748ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [760ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [764ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [767ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [802ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [805ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [807ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [840ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [843ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [845ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [858ms] refreshing cache from master, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [862ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [865ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [867ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [980ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [981ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [986ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [987ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [988ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1217ms] refreshing cache from master, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1225ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1227ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1228ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1425ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1426ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3897ms] refreshing cache from master, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [3899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [3905ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [3906ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3906ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [6898ms] refreshing cache from master, [6898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [6899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [6900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [6900ms] too many traces: truncated at 100 traces, deferred=Deferred@1335312408(state=PENDING, result=null, callback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.76.62:33369
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29546), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [215ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [219ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [710ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [718ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [724ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [733ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [734ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [745ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [746ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [748ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [760ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [764ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [767ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [802ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [805ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [807ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [840ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [843ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [845ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [858ms] refreshing cache from master, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [862ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [865ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [867ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [980ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [981ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [986ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [987ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [988ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1217ms] refreshing cache from master, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1225ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1227ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1228ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1425ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1426ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3897ms] refreshing cache from master, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [3899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [3905ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [3906ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3906ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [6898ms] refreshing cache from master, [6898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [6899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [6900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [6900ms] too many traces: truncated at 100 traces, deferred=Deferred@1335312408(state=PENDING, result=null, callback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.76.62:33369
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
13:47:37.692 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
13:47:37.692 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29546), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [215ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [219ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [710ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [718ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [724ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [733ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [734ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [745ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [746ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [748ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [760ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [764ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [767ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [802ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [805ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [807ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [840ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [843ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [845ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [858ms] refreshing cache from master, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [862ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [865ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [867ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [980ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [981ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [986ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [987ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [988ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1217ms] refreshing cache from master, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1225ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1227ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1228ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1425ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1426ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3897ms] refreshing cache from master, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [3899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [3905ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [3906ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3906ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [6898ms] refreshing cache from master, [6898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [6899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [6900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [6900ms] too many traces: truncated at 100 traces, deferred=Deferred@1335312408(state=PENDING, result=null, callback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.76.62:33369
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29546), Traces: [0ms] refreshing cache from master, [36ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [215ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [219ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [710ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [718ms] refreshing cache from master, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [721ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [724ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [732ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [733ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [734ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [740ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [745ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [746ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [748ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [760ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [764ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [767ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [768ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [797ms] refreshing cache from master, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [799ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [802ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [805ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [807ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [807ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [837ms] refreshing cache from master, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [838ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [840ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [840ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [843ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [845ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [846ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [858ms] refreshing cache from master, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [860ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [862ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [865ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [867ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [868ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [977ms] refreshing cache from master, [978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [980ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [981ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [982ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [986ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [987ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [988ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1217ms] refreshing cache from master, [1218ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1221ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1225ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1227ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1228ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1417ms] refreshing cache from master, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [1419ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [1425ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [1426ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2457ms] refreshing cache from master, [2458ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [2459ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [2460ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [2462ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [2464ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [2464ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3897ms] refreshing cache from master, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [3899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [3900ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.61:33787: Network error: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787, [3905ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.60:34005: OK, [3906ms] Sub RPC ConnectToMaster: received response from server master-127.18.76.62:33369: Network error: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [3906ms] delaying RPC due to: Service unavailable: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369, [6898ms] refreshing cache from master, [6898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.61:33787, [6899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.62:33369, [6900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.18.76.60:34005, [6900ms] too many traces: truncated at 100 traces, deferred=Deferred@1335312408(state=PENDING, result=null, callback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995), errback=(continuation of Deferred@208174907 after retry RPC after error@104037575) -> (continuation of Deferred@1361492537 after retry RPC after error@608479109) -> (continuation of Deferred@1796900241 after retry RPC after error@1985091953) -> (continuation of Deferred@314671856 after retry RPC after error@322463650) -> (continuation of Deferred@749821529 after retry RPC after error@1246642991) -> (continuation of Deferred@1409715860 after retry RPC after error@81000129) -> (continuation of Deferred@582366100 after retry RPC after error@442059734) -> (continuation of Deferred@2051740711 after retry RPC after error@1630896209) -> (continuation of Deferred@1913402736 after retry RPC after error@584828700) -> (continuation of Deferred@1250636 after retry RPC after error@1307130391) -> (continuation of Deferred@1583161711 after retry RPC after error@583200415) -> (continuation of Deferred@979576574 after retry RPC after error@1046387239) -> (continuation of Deferred@163549914 after retry RPC after error@1010574180) -> (continuation of Deferred@525055691 after retry RPC after error@1427015847) -> (continuation of Deferred@542304670 after retry RPC after error@2099825139) -> (continuation of Deferred@1058526579 after retry RPC after error@511523206) -> (continuation of Deferred@701771214 after retry RPC after error@310910550) -> (continuation of Deferred@322301241 after retry RPC after error@1693669328) -> (continuation of Deferred@1824636125 after retry RPC after error@1055487421) -> (continuation of Deferred@34151292 after retry RPC after error@2092597562) -> (continuation of Deferred@494726436 after retry RPC after error@1758279595) -> (continuation of Deferred@575996725 after retry RPC after error@181335316) -> (continuation of Deferred@1881226404 after retry RPC after error@1909468516) -> (continuation of Deferred@2073482432 after retry RPC after error@101578995)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.18.76.61:33787,127.18.76.62:33369,127.18.76.60:34005) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.61:33787(127.18.76.61:33787): Connection refused: /127.18.76.61:33787,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.18.76.62:33369(127.18.76.62:33369): Connection refused: /127.18.76.62:33369
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.18.76.62:33369
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)