15:35:37.466 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:35:37.478 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskE66ITT/test-tmp
15:35:37.480 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:35:37.483 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskE66ITT/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:35:37.750 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:35:37.900 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:35:37.901 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:35:37.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:37.904976 374 mini_oidc.cc:152] Starting JWKS server
15:35:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:37.908972 374 webserver.cc:460] Webserver started at http://127.0.0.1:33155/ using document root <none> and password file <none>
15:35:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:37.909471 374 mini_oidc.cc:189] Starting OIDC Discovery server
15:35:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:37.911772 374 webserver.cc:460] Webserver started at http://127.0.0.1:37371/ using document root <none> and password file <none>
15:35:37.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T15:35:37Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:35:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T15:35:37Z Disabled control of system clock
15:35:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:37.960527 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/wal
15:35:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data
15:35:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:37.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/logs
15:35:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data/info.pb
15:35:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:37.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:33599
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33155/jwks.json with env {}
15:35:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.275756 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.276369 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:38.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.276991 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.315064 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:35:38.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.315380 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.315570 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:35:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.315747 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:35:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.316068 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:35:38.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.316239 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33155/jwks.json
15:35:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:38.351634 386 master_runner.cc:386] Master server non-default flags:
15:35:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/wal
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:33599
15:35:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33155/jwks.json
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data/info.pb
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
15:35:38.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/logs
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:38.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:38.353044 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:38.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:38.354749 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:38.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.374405 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:39.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.374413 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:39.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:38.375093 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:39.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:38.375602 386 server_base.cc:1034] running on GCE node
15:35:39.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.559912 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:39.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.563488 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:39.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.564903 386 hybrid_clock.cc:648] HybridClock initialized: now 1790696139564860 us; error 66 us; skew 500 ppm
15:35:39.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.565689 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:39.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.578348 386 webserver.cc:460] Webserver started at http://127.0.93.190:44055/ using document root <none> and password file <none>
15:35:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.579315 386 fs_manager.cc:359] Metadata directory not provided
15:35:39.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.579504 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.579898 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.584649 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data/instance:
15:35:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d"
15:35:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:39 on dist-test-slave-cq88"
15:35:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.585841 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/wal/instance:
15:35:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d"
15:35:39.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:39 on dist-test-slave-cq88"
15:35:39.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.597281 386 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.006s sys 0.003s
15:35:39.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.603364 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:39.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.604601 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
15:35:39.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.604897 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/wal
15:35:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d"
15:35:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:39 on dist-test-slave-cq88"
15:35:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.605191 386 fs_report.cc:383] FS layout report
15:35:39.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/wal
15:35:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/wal
15:35:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data/data
15:35:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:39.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:39.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:39.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.657805 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.659292 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.659754 386 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:39.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.738118 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:33599
15:35:39.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.738205 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:33599 every 8 connection(s)
15:35:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.740875 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/data/info.pb
15:35:39.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.747191 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:35:39.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.747553 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 386
15:35:39.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.747910 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-0/wal/instance
15:35:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.751080 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/wal
15:35:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/logs
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data/info.pb
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:39.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:39.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:34345
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:39.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:39.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:39.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33155/jwks.json with env {}
15:35:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.761125 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } has no permanent_uuid. Determining permanent_uuid...
15:35:39.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.796375 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } has no permanent_uuid. Determining permanent_uuid...
15:35:39.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:39.799047 403 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:39.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:39.804138 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34345: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.853713 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } attempt: 1
15:35:39.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:39.857892 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34345: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:39.926367 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } attempt: 2
15:35:39.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:39.930436 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34345: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:40.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:40.021903 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } attempt: 3
15:35:40.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.026091 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34345: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:40.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.086005 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:40.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.086581 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:40.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.087145 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.120747 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:35:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.121037 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:40.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.121224 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:35:40.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.121398 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:35:40.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.121580 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:35:40.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.121783 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33155/jwks.json
15:35:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:40.157442 456 master_runner.cc:386] Master server non-default flags:
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/wal
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:40.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:34345
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33155/jwks.json
15:35:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data/info.pb
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/logs
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:35:40.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:40.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:40.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:40.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:40.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:40.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:40.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:40.158627 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:40.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:40.160741 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:40.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:40.169581 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } attempt: 4
15:35:40.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.175048 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34345: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:40.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.183609 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:40.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:40.474620 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } attempt: 5
15:35:40.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.480724 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34345: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.028302 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } attempt: 6
15:35:41.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.038789 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34345: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34345: connect: Connection refused (error 111)
15:35:41.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.186797 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:41.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:40.184099 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:41.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.461131 467 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1276 milliseconds
15:35:41.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.461251 456 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:35:41.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.462791 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.466403 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:41.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.467844 456 hybrid_clock.cc:648] HybridClock initialized: now 1790696141467798 us; error 66 us; skew 500 ppm
15:35:41.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.468644 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:41.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.475075 456 webserver.cc:460] Webserver started at http://127.0.93.189:46083/ using document root <none> and password file <none>
15:35:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.476059 456 fs_manager.cc:359] Metadata directory not provided
15:35:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.476246 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.476611 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:41.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.481418 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data/instance:
15:35:41.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb2ccf3776e64dee9c187b6255fb80e2"
15:35:41.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:41 on dist-test-slave-cq88"
15:35:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.483023 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/wal/instance:
15:35:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb2ccf3776e64dee9c187b6255fb80e2"
15:35:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:41 on dist-test-slave-cq88"
15:35:41.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.490007 456 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.001s sys 0.005s
15:35:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.495833 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.497087 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:35:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.497409 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/wal
15:35:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb2ccf3776e64dee9c187b6255fb80e2"
15:35:41.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:41 on dist-test-slave-cq88"
15:35:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.497723 456 fs_report.cc:383] FS layout report
15:35:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:41.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/wal
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/wal
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data/data
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:41.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:41.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.541060 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:41.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.542477 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:41.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.542896 456 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:41.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.621079 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:34345
15:35:41.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.621157 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:34345 every 8 connection(s)
15:35:41.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.623751 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/data/info.pb
15:35:41.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.629418 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:35:41.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.631281 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 456
15:35:41.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.631654 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-1/wal/instance
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.633944 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/wal
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:41.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/logs
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data/info.pb
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:41.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:38031
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:41.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33155/jwks.json with env {}
15:35:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.643106 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } has no permanent_uuid. Determining permanent_uuid...
15:35:41.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.671204 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } has no permanent_uuid. Determining permanent_uuid...
15:35:41.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.688038 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } has no permanent_uuid. Determining permanent_uuid...
15:35:41.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.691103 480 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.695258 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.744815 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 1
15:35:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.749449 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:41.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.817888 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 2
15:35:41.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.821520 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:41.913015 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 3
15:35:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.917145 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.963543 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.964097 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.964538 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.995048 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:35:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.995429 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:41.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.995729 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:35:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.995934 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:35:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.996111 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:35:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:41.996325 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33155/jwks.json
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.033519 531 master_runner.cc:386] Master server non-default flags:
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/wal
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:38031
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33155/jwks.json
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data/info.pb
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/logs
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:42.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:42.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:42.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:42.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.034858 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.037140 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:42.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.059525 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:42.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.060552 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 4
15:35:42.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.065610 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.099619 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } attempt: 7
15:35:42.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.122539 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } has no permanent_uuid. Determining permanent_uuid...
15:35:42.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.131803 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.190533 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 1
15:35:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.196870 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.278458 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 2
15:35:42.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.284386 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.365175 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 5
15:35:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.370117 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 3
15:35:42.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.371693 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.376678 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.517213 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 4
15:35:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.523685 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.807294 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 5
15:35:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.813294 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:42.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:42.919224 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 6
15:35:42.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.930169 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:43.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.060878 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:43.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:42.059727 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:43.302843 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1241 milliseconds
15:35:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.302958 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:35:43.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.304188 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:43.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.306522 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:43.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.307852 531 hybrid_clock.cc:648] HybridClock initialized: now 1790696143307838 us; error 49 us; skew 500 ppm
15:35:43.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.308619 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:43.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.314985 531 webserver.cc:460] Webserver started at http://127.0.93.188:42425/ using document root <none> and password file <none>
15:35:43.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.318334 531 fs_manager.cc:359] Metadata directory not provided
15:35:43.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.318550 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:43.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.318934 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:43.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.323764 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data/instance:
15:35:43.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfee1ab358e74145a1eae527b3af3e64"
15:35:43.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:43 on dist-test-slave-cq88"
15:35:43.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.324766 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/wal/instance:
15:35:43.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfee1ab358e74145a1eae527b3af3e64"
15:35:43.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:43 on dist-test-slave-cq88"
15:35:43.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.332312 531 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
15:35:43.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.337505 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:43.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.338487 531 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:35:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.338832 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/wal
15:35:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bfee1ab358e74145a1eae527b3af3e64"
15:35:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:43 on dist-test-slave-cq88"
15:35:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.339185 531 fs_report.cc:383] FS layout report
15:35:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:43.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/wal
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/wal
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data/data
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:43.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.365870 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 6
15:35:43.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:43.372334 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:38031: Network error: Client connection negotiation failed: client connection to 127.0.93.188:38031: connect: Connection refused (error 111)
15:35:43.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.378259 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.379899 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.380429 531 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.461925 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:38031
15:35:43.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.462030 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:38031 every 8 connection(s)
15:35:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.464507 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/data/info.pb
15:35:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.468242 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 531
15:35:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.468658 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/master-2/wal/instance
15:35:43.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.470527 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:35:43.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.488533 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } has no permanent_uuid. Determining permanent_uuid...
15:35:43.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.518550 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } has no permanent_uuid. Determining permanent_uuid...
15:35:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.530498 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } has no permanent_uuid. Determining permanent_uuid...
15:35:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.549358 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64: Bootstrap starting.
15:35:43.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.557076 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64: Neither blocks nor log segments found. Creating new log.
15:35:43.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.558889 609 log.cc:824] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64: Log is configured to *not* fsync() on all Append() calls
15:35:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.563674 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64: No bootstrap required, opened a new log
15:35:43.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.587457 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.588371 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:35:43.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.588685 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: bfee1ab358e74145a1eae527b3af3e64, State: Initialized, Role: FOLLOWER
15:35:43.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.589934 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:43.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.592571 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:43.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.593309 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: This master's current role is: FOLLOWER
15:35:43.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.594676 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: configured and running, proceeding with master startup.
15:35:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.622889 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:35:43.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.623751 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.633623 618 leader_election.cc:288] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers cbf6d93fdd104a7b9dc0cf97c3dead1d (127.0.93.190:33599), cb2ccf3776e64dee9c187b6255fb80e2 (127.0.93.189:34345)
15:35:43.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:43.636758 563 tablet.cc:2369] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:35:43.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:43.642840 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:35:43.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:43.643445 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:35:43.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.622360 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:33599: OK, instance_id { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" instance_seqno: 1790696139715849 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:35:43.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.666235 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:35:43.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:43.990728 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 7
15:35:44.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.014868 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: Bootstrap starting.
15:35:44.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.022951 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: Neither blocks nor log segments found. Creating new log.
15:35:44.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.025437 529 log.cc:824] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: Log is configured to *not* fsync() on all Append() calls
15:35:44.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.042840 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: No bootstrap required, opened a new log
15:35:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.076272 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.077139 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:35:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.077414 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb2ccf3776e64dee9c187b6255fb80e2, State: Initialized, Role: FOLLOWER
15:35:44.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.078691 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.082046 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.083137 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [sys.catalog]: This master's current role is: FOLLOWER
15:35:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.083930 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [sys.catalog]: configured and running, proceeding with master startup.
15:35:44.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.084100 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfee1ab358e74145a1eae527b3af3e64" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" is_pre_election: true
15:35:44.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.084981 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate bfee1ab358e74145a1eae527b3af3e64 in term 0.
15:35:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.086992 556 leader_election.cc:302] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bfee1ab358e74145a1eae527b3af3e64, cb2ccf3776e64dee9c187b6255fb80e2; no voters:
15:35:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.087921 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:35:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.088280 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:35:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.088593 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 0 FOLLOWER]: Advancing to term 1
15:35:44.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.096359 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:44.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.100410 618 leader_election.cc:288] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [CANDIDATE]: Term 1 election: Requested vote from peers cbf6d93fdd104a7b9dc0cf97c3dead1d (127.0.93.190:33599), cb2ccf3776e64dee9c187b6255fb80e2 (127.0.93.189:34345)
15:35:44.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.101635 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfee1ab358e74145a1eae527b3af3e64" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cb2ccf3776e64dee9c187b6255fb80e2"
15:35:44.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.102193 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [term 0 FOLLOWER]: Advancing to term 1
15:35:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.109195 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bfee1ab358e74145a1eae527b3af3e64 in term 1.
15:35:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.110358 556 leader_election.cc:302] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: bfee1ab358e74145a1eae527b3af3e64, cb2ccf3776e64dee9c187b6255fb80e2; no voters:
15:35:44.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.111142 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 1 FOLLOWER]: Leader election won for term 1
15:35:44.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.113667 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [term 1 LEADER]: Becoming Leader. State: Replica: bfee1ab358e74145a1eae527b3af3e64, State: Running, Role: LEADER
15:35:44.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.114590 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.128618 641 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfee1ab358e74145a1eae527b3af3e64. Latest consensus state: current_term: 1 leader_uuid: "bfee1ab358e74145a1eae527b3af3e64" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.129798 641 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: This master's current role is: LEADER
15:35:44.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.135257 646 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
15:35:44.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.143425 646 catalog_manager.cc:1484] Initializing Kudu cluster ID...
15:35:44.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.174422 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [term 1 FOLLOWER]: Refusing update from remote peer bfee1ab358e74145a1eae527b3af3e64: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:35:44.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.176621 641 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:35:44.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:44.180928 647 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:35:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:44.181744 647 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:35:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.151089 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:33599: OK, instance_id { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" instance_seqno: 1790696139715849 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:35:44.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.196244 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:35:44.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.219655 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfee1ab358e74145a1eae527b3af3e64. Latest consensus state: current_term: 1 leader_uuid: "bfee1ab358e74145a1eae527b3af3e64" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.221321 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [sys.catalog]: This master's current role is: FOLLOWER
15:35:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.230639 650 mvcc.cc:205] Tried to move back new op lower bound from 7334691406505295872 to 7334691406309224448. Current Snapshot: MvccSnapshot[applied={T|T < 7334691406505295872}]
15:35:44.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.238621 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bfee1ab358e74145a1eae527b3af3e64" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.239509 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: This master's current role is: LEADER
15:35:44.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.245174 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bfee1ab358e74145a1eae527b3af3e64" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.247054 646 catalog_manager.cc:1347] Generated new cluster ID: 56667bb03dba432bbafcdda5c4a9aeaa
15:35:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.247342 646 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
15:35:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.246755 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2 [sys.catalog]: This master's current role is: FOLLOWER
15:35:44.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.285956 646 catalog_manager.cc:1370] Generated new certificate authority record
15:35:44.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.293342 646 catalog_manager.cc:1504] Loading token signing keys...
15:35:44.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.328179 646 catalog_manager.cc:5868] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64: Generated new TSK 0
15:35:44.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.329403 646 catalog_manager.cc:1514] Initializing in-progress tserver states...
15:35:44.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.352403 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/wal
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/logs
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data/info.pb
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:44.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:44.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:35:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.405689 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } attempt: 7
15:35:44.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.446483 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: Bootstrap starting.
15:35:44.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.474465 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: Neither blocks nor log segments found. Creating new log.
15:35:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.489327 454 log.cc:824] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: Log is configured to *not* fsync() on all Append() calls
15:35:44.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.508422 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: No bootstrap required, opened a new log
15:35:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.588861 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.590680 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:35:44.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.591009 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cbf6d93fdd104a7b9dc0cf97c3dead1d, State: Initialized, Role: FOLLOWER
15:35:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.592617 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } }
15:35:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.610270 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [sys.catalog]: configured and running, proceeding with master startup.
15:35:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.611019 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [term 0 FOLLOWER]: Advancing to term 1
15:35:44.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.611922 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfee1ab358e74145a1eae527b3af3e64" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d"
15:35:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.611205 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "bfee1ab358e74145a1eae527b3af3e64" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" is_pre_election: true
15:35:44.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.622411 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [term 1 FOLLOWER]: Refusing update from remote peer bfee1ab358e74145a1eae527b3af3e64: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:35:44.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.628108 618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
15:35:44.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.628880 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.629654 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [sys.catalog]: This master's current role is: FOLLOWER
15:35:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:44.673954 408 tablet.cc:2369] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:35:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.703958 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [sys.catalog]: SysCatalogTable state changed. Reason: New leader bfee1ab358e74145a1eae527b3af3e64. Latest consensus state: current_term: 1 leader_uuid: "bfee1ab358e74145a1eae527b3af3e64" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.704799 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [sys.catalog]: This master's current role is: FOLLOWER
15:35:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.704389 661 sys_catalog.cc:455] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "bfee1ab358e74145a1eae527b3af3e64" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:44.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.705129 661 sys_catalog.cc:458] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d [sys.catalog]: This master's current role is: FOLLOWER
15:35:44.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:44.754344 673 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:35:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.027940 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "bfee1ab358e74145a1eae527b3af3e64" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "cbf6d93fdd104a7b9dc0cf97c3dead1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 33599 } } peers { permanent_uuid: "cb2ccf3776e64dee9c187b6255fb80e2" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34345 } } peers { permanent_uuid: "bfee1ab358e74145a1eae527b3af3e64" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 38031 } } }
15:35:45.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.029543 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P bfee1ab358e74145a1eae527b3af3e64 [sys.catalog]: This master's current role is: LEADER
15:35:45.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.085366 654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:45.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.086357 654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:45.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.087144 654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:45.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.141429 654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:45.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.142833 654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
15:35:45.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.193135 647 catalog_manager.cc:1259] Loaded cluster ID: 56667bb03dba432bbafcdda5c4a9aeaa
15:35:45.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.193740 647 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: loading cluster ID for follower catalog manager: success
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.203529 654 tablet_server_runner.cc:78] Tablet server non-default flags:
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/wal
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data/info.pb
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/logs
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.205070 654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:45.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.207139 654 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:45.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.208969 647 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: acquiring CA information for follower catalog manager: success
15:35:45.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.215396 647 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cb2ccf3776e64dee9c187b6255fb80e2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:35:45.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.234524 683 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.768060 673 catalog_manager.cc:1259] Loaded cluster ID: 56667bb03dba432bbafcdda5c4a9aeaa
15:35:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.773825 673 catalog_manager.cc:1552] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: loading cluster ID for follower catalog manager: success
15:35:45.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.797125 673 catalog_manager.cc:1574] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: acquiring CA information for follower catalog manager: success
15:35:45.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:45.821310 673 catalog_manager.cc:1602] T 00000000000000000000000000000000 P cbf6d93fdd104a7b9dc0cf97c3dead1d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:35:46.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.264053 685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:46.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:46.865180 684 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1625 milliseconds
15:35:46.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.865224 654 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:35:46.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:45.236153 682 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.870097 654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:46.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.884927 654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:46.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.886368 654 hybrid_clock.cc:648] HybridClock initialized: now 1790696146886317 us; error 66 us; skew 500 ppm
15:35:46.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.887482 654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:46.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.918097 654 webserver.cc:460] Webserver started at http://127.0.93.129:37849/ using document root <none> and password file <none>
15:35:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.919462 654 fs_manager.cc:359] Metadata directory not provided
15:35:46.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.919762 654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.920353 654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:46.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.927523 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data/instance:
15:35:46.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd47cdd14279404f86f7e84162c43f1b"
15:35:46.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:46 on dist-test-slave-cq88"
15:35:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.929150 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/wal/instance:
15:35:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd47cdd14279404f86f7e84162c43f1b"
15:35:46.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:46 on dist-test-slave-cq88"
15:35:46.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.955812 654 fs_manager.cc:691] Time spent creating directory manager: real 0.026s user 0.014s sys 0.013s
15:35:46.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.969506 693 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.970846 654 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.007s sys 0.001s
15:35:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.971254 654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/wal
15:35:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bd47cdd14279404f86f7e84162c43f1b"
15:35:46.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:46 on dist-test-slave-cq88"
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:46.971700 654 fs_report.cc:383] FS layout report
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/wal
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/wal
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data/data
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:46.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:47.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.051782 654 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.053753 654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.054328 654 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:47.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.063745 654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:35:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.075788 654 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:35:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.076094 654 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:35:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.076391 654 ts_tablet_manager.cc:610] Registered 0 tablets
15:35:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.076618 654 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:35:47.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.526268 654 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:33013
15:35:47.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.526423 805 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:33013 every 8 connection(s)
15:35:47.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.529897 654 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/data/info.pb
15:35:47.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.534205 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 654
15:35:47.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.534691 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-0/wal/instance
15:35:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.558365 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:47.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:47.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/wal
15:35:47.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data
15:35:47.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/logs
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data/info.pb
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
15:35:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
15:35:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
15:35:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:35:47.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.598691 806 heartbeater.cc:347] Connected to a master server at 127.0.93.189:34345
15:35:47.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.601845 806 heartbeater.cc:464] Registering TS with master...
15:35:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.603200 806 heartbeater.cc:511] Master 127.0.93.189:34345 requested a full tablet report, sending...
15:35:47.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.608144 494 ts_manager.cc:194] Registered new tserver with Master: bd47cdd14279404f86f7e84162c43f1b (127.0.93.129:33013)
15:35:47.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.615916 811 heartbeater.cc:347] Connected to a master server at 127.0.93.190:33599
15:35:47.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.616298 811 heartbeater.cc:464] Registering TS with master...
15:35:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.617134 811 heartbeater.cc:511] Master 127.0.93.190:33599 requested a full tablet report, sending...
15:35:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.620220 418 ts_manager.cc:194] Registered new tserver with Master: bd47cdd14279404f86f7e84162c43f1b (127.0.93.129:33013)
15:35:47.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.623631 807 heartbeater.cc:347] Connected to a master server at 127.0.93.188:38031
15:35:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.623979 807 heartbeater.cc:464] Registering TS with master...
15:35:47.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.624861 807 heartbeater.cc:511] Master 127.0.93.188:38031 requested a full tablet report, sending...
15:35:47.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.627700 574 ts_manager.cc:194] Registered new tserver with Master: bd47cdd14279404f86f7e84162c43f1b (127.0.93.129:33013)
15:35:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:47.630964 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:35905
15:35:47.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:47.938319 810 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:47.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:47.938944 810 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:47.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:47.939589 810 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:47.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:47.971061 810 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:47.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:47.972072 810 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
15:35:48.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:48.009191 810 tablet_server_runner.cc:78] Tablet server non-default flags:
15:35:48.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:48.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/wal
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data/info.pb
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:48.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/logs
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:48.010483 810 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:48.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:48.012099 810 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:48.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:48.039551 824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:48.636178 807 heartbeater.cc:503] Master 127.0.93.188:38031 was elected leader, sending a full tablet report...
15:35:49.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:48.040205 823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:49.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:48.041754 810 server_base.cc:1034] running on GCE node
15:35:49.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:48.041884 826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.260242 810 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.263139 810 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:49.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.264703 810 hybrid_clock.cc:648] HybridClock initialized: now 1790696149264670 us; error 56 us; skew 500 ppm
15:35:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.265467 810 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:49.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.272071 810 webserver.cc:460] Webserver started at http://127.0.93.130:40177/ using document root <none> and password file <none>
15:35:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.272979 810 fs_manager.cc:359] Metadata directory not provided
15:35:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.273231 810 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:49.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.273746 810 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:49.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.278468 810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data/instance:
15:35:49.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a19656167cf040daa7cf485643d6b149"
15:35:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:49 on dist-test-slave-cq88"
15:35:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.279592 810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/wal/instance:
15:35:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a19656167cf040daa7cf485643d6b149"
15:35:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:49 on dist-test-slave-cq88"
15:35:49.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.286818 810 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:35:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.292629 834 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.293880 810 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:35:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.294157 810 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/wal
15:35:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a19656167cf040daa7cf485643d6b149"
15:35:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:49 on dist-test-slave-cq88"
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.294533 810 fs_report.cc:383] FS layout report
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/wal
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/wal
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data/data
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:49.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:49.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.344390 810 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:49.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.345839 810 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:49.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.346300 810 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:49.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.349077 810 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:35:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.352998 810 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:35:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.353210 810 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:35:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.353410 810 ts_tablet_manager.cc:610] Registered 0 tablets
15:35:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.353544 810 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:35:49.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.512547 810 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:37175
15:35:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.512696 946 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:37175 every 8 connection(s)
15:35:49.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.515153 810 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/data/info.pb
15:35:49.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.516459 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 810
15:35:49.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.517011 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-1/wal/instance
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.554775 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/wal
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/logs
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data/info.pb
15:35:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:35:49.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.588322 948 heartbeater.cc:347] Connected to a master server at 127.0.93.188:38031
15:35:49.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.588827 948 heartbeater.cc:464] Registering TS with master...
15:35:49.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.588961 947 heartbeater.cc:347] Connected to a master server at 127.0.93.189:34345
15:35:49.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.589320 947 heartbeater.cc:464] Registering TS with master...
15:35:49.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.590764 948 heartbeater.cc:511] Master 127.0.93.188:38031 requested a full tablet report, sending...
15:35:49.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.590943 947 heartbeater.cc:511] Master 127.0.93.189:34345 requested a full tablet report, sending...
15:35:49.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.590989 949 heartbeater.cc:347] Connected to a master server at 127.0.93.190:33599
15:35:49.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.592252 949 heartbeater.cc:464] Registering TS with master...
15:35:49.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.593082 949 heartbeater.cc:511] Master 127.0.93.190:33599 requested a full tablet report, sending...
15:35:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.593501 574 ts_manager.cc:194] Registered new tserver with Master: a19656167cf040daa7cf485643d6b149 (127.0.93.130:37175)
15:35:49.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.593959 494 ts_manager.cc:194] Registered new tserver with Master: a19656167cf040daa7cf485643d6b149 (127.0.93.130:37175)
15:35:49.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.595760 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:58453
15:35:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.596560 418 ts_manager.cc:194] Registered new tserver with Master: a19656167cf040daa7cf485643d6b149 (127.0.93.130:37175)
15:35:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:49.908800 957 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:49.909272 957 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:49.909821 957 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:49.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:49.940577 957 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:49.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:49.941409 957 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.975996 957 tablet_server_runner.cc:78] Tablet server non-default flags:
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43469
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/wal
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data/info.pb
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:33599,127.0.93.189:34345,127.0.93.188:38031
15:35:49.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/logs
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:49.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.977368 957 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:49.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:49.979194 957 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:50.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:50.001679 963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:50.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:50.600718 948 heartbeater.cc:503] Master 127.0.93.188:38031 was elected leader, sending a full tablet report...
15:35:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:50.001729 964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:50.001686 966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:51.211745 965 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:35:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.211854 957 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:35:51.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.215675 957 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:51.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.217723 957 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:51.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.219044 957 hybrid_clock.cc:648] HybridClock initialized: now 1790696151219001 us; error 64 us; skew 500 ppm
15:35:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.219956 957 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:51.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.226043 957 webserver.cc:460] Webserver started at http://127.0.93.131:34247/ using document root <none> and password file <none>
15:35:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.226961 957 fs_manager.cc:359] Metadata directory not provided
15:35:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.227147 957 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:51.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.227605 957 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.232506 957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data/instance:
15:35:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0120c58ee62b47658587d4a4bbadfab1"
15:35:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:51 on dist-test-slave-cq88"
15:35:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.233541 957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/wal/instance:
15:35:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0120c58ee62b47658587d4a4bbadfab1"
15:35:51.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:51 on dist-test-slave-cq88"
15:35:51.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.240514 957 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
15:35:51.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.245869 974 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.246963 957 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:35:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.247327 957 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/wal
15:35:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0120c58ee62b47658587d4a4bbadfab1"
15:35:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:51 on dist-test-slave-cq88"
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.247653 957 fs_report.cc:383] FS layout report
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/wal
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/wal
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data/data
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:51.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.291145 957 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.292621 957 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:51.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.293020 957 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:51.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.295584 957 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:35:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.299698 957 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:35:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.299880 957 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:35:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.300071 957 ts_tablet_manager.cc:610] Registered 0 tablets
15:35:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.300199 957 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:35:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.449928 957 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:42643
15:35:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.450039 1086 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:42643 every 8 connection(s)
15:35:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.452312 957 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/data/info.pb
15:35:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.457587 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 957
15:35:51.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.458029 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster188870525819944349/ts-2/wal/instance
15:35:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.518234 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.190:33599
15:35:51.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.518751 1089 heartbeater.cc:464] Registering TS with master...
15:35:51.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.519936 1089 heartbeater.cc:511] Master 127.0.93.190:33599 requested a full tablet report, sending...
15:35:51.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.521193 1087 heartbeater.cc:347] Connected to a master server at 127.0.93.189:34345
15:35:51.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.521539 1087 heartbeater.cc:464] Registering TS with master...
15:35:51.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.522377 1087 heartbeater.cc:511] Master 127.0.93.189:34345 requested a full tablet report, sending...
15:35:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.525050 418 ts_manager.cc:194] Registered new tserver with Master: 0120c58ee62b47658587d4a4bbadfab1 (127.0.93.131:42643)
15:35:51.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.525460 494 ts_manager.cc:194] Registered new tserver with Master: 0120c58ee62b47658587d4a4bbadfab1 (127.0.93.131:42643)
15:35:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.527345 1088 heartbeater.cc:347] Connected to a master server at 127.0.93.188:38031
15:35:51.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.528117 1088 heartbeater.cc:464] Registering TS with master...
15:35:51.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.528959 1088 heartbeater.cc:511] Master 127.0.93.188:38031 requested a full tablet report, sending...
15:35:51.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.531092 574 ts_manager.cc:194] Registered new tserver with Master: 0120c58ee62b47658587d4a4bbadfab1 (127.0.93.131:42643)
15:35:51.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.532717 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:40721
15:35:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:51.542938 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
15:35:51.553 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:35:51.555 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:35:51.565 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 33599
}
bound_http_address {
host: "127.0.93.190"
port: 44055
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 34345
}
bound_http_address {
host: "127.0.93.189"
port: 46083
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 38031
}
bound_http_address {
host: "127.0.93.188"
port: 42425
}
}
}
15:35:51.571 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:35:51.574 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 33013
}
bound_http_address {
host: "127.0.93.129"
port: 37849
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 37175
}
bound_http_address {
host: "127.0.93.130"
port: 40177
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 42643
}
bound_http_address {
host: "127.0.93.131"
port: 34247
}
}
}
15:35:51.575 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:35:51.769 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:35:51.888 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskE66ITT/test-tmp
15:35:51.889 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:35:51.889 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskE66ITT/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:35:51.893 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:35:52.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:52.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:35:52.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:52.239681 1098 mini_oidc.cc:152] Starting JWKS server
15:35:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:52.244475 1098 webserver.cc:460] Webserver started at http://127.0.0.1:36831/ using document root <none> and password file <none>
15:35:52.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:52.245074 1098 mini_oidc.cc:189] Starting OIDC Discovery server
15:35:52.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:52.247179 1098 webserver.cc:460] Webserver started at http://127.0.0.1:36247/ using document root <none> and password file <none>
15:35:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:35:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/principal' for realm 'KRBTEST.COM',
15:35:52.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:35:52.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:52 dist-test-slave-cq88 krb5kdc[1111](info): setting up network...
15:35:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:35:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:52 dist-test-slave-cq88 krb5kdc[1111](info): set up 2 sockets
15:35:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:52 dist-test-slave-cq88 krb5kdc[1111](info): commencing operation
15:35:52.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:35:52.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:52.537209 1088 heartbeater.cc:503] Master 127.0.93.188:38031 was elected leader, sending a full tablet report...
15:35:53.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.385360 1098 mini_kdc.cc:121] Time spent starting KDC: real 1.113s user 0.008s sys 0.013s
15:35:53.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:35:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:35:53.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:35:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:35:53.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:35:53.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:35:53.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:35:53.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:35:53.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:35:53.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:35:53.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/test-user.keytab.
15:35:53.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/test-user.keytab.
15:35:53.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:53 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696153, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:35:53.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:35:53.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T15:35:53Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:35:53.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T15:35:53Z Disabled control of system clock
15:35:53.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.546350 943 debug-util.cc:398] Leaking SignalData structure 0x7b08000af400 after lost signal to thread 818
15:35:53.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.547660 943 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd40 after lost signal to thread 946
15:35:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.190@KRBTEST.COM; defaulting to no policy
15:35:53.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:35:53.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.190@KRBTEST.COM" created.
15:35:53.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:35:53.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.190.keytab.
15:35:53.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.190.keytab.
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:53.600725 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/logs
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/info.pb
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:33879
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.190.keytab
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:35:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:35:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.938920 1124 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.939519 1124 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.940115 1124 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.972589 1124 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:35:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.972818 1124 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:35:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.973078 1124 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.973271 1124 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:35:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.973457 1124 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:35:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.973636 1124 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:35:53.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:53.973783 1124 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36831/jwks.json
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:54.011905 1124 master_runner.cc:386] Master server non-default flags:
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.190.keytab
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:54.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:33879
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/info.pb
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/logs
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:54.013172 1124 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:54.014266 1124 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:54.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:54.037451 1131 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:55.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:54.038721 1133 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:55.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:54.038865 1124 server_base.cc:1034] running on GCE node
15:35:55.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:54.037928 1130 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:55.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.330406 1124 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.333657 1124 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:55.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.335351 1124 hybrid_clock.cc:648] HybridClock initialized: now 1790696155335308 us; error 67 us; skew 500 ppm
15:35:55.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.336227 1124 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:55.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:55 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696155, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:35:55.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.342865 1124 init.cc:377] Logged in from keytab as kudu/127.1.18.190@KRBTEST.COM (short username kudu)
15:35:55.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.349509 1124 webserver.cc:460] Webserver started at http://127.1.18.190:43303/ using document root <none> and password file <none>
15:35:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.350847 1124 fs_manager.cc:359] Metadata directory not provided
15:35:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.351142 1124 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.351799 1124 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.359750 1124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/instance:
15:35:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "026b7023b53e4564a8e30fceefb3255b"
15:35:55.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:55 on dist-test-slave-cq88"
15:35:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.361238 1124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal/instance:
15:35:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "026b7023b53e4564a8e30fceefb3255b"
15:35:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:55 on dist-test-slave-cq88"
15:35:55.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.371022 1124 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.004s
15:35:55.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.378021 1142 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:55.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.379582 1124 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.007s sys 0.000s
15:35:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.379968 1124 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:35:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "026b7023b53e4564a8e30fceefb3255b"
15:35:55.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:55 on dist-test-slave-cq88"
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.380414 1124 fs_report.cc:383] FS layout report
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/data
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:55.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:55.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.418742 1124 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:55.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.424295 1124 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:55.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.424692 1124 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:55.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.511260 1124 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.190:33879
15:35:55.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.511785 1193 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.190:33879 every 8 connection(s)
15:35:55.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.514856 1124 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/info.pb
15:35:55.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.521116 1194 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:35:55.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.523284 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 1124
15:35:55.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.523766 1098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal/instance
15:35:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.534924 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } has no permanent_uuid. Determining permanent_uuid...
15:35:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.189@KRBTEST.COM; defaulting to no policy
15:35:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:35:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.189@KRBTEST.COM" created.
15:35:55.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:55 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696155, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
15:35:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:35:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.189.keytab.
15:35:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.189.keytab.
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.579993 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/logs
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/info.pb
15:35:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:32777
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.189.keytab
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:35:55.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:35:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:35:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.580786 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } has no permanent_uuid. Determining permanent_uuid...
15:35:55.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.584088 1144 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:55.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.590021 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:32777: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.639530 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } attempt: 1
15:35:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.643451 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:32777: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:55.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.711887 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } attempt: 2
15:35:55.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.715958 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:32777: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.807411 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } attempt: 3
15:35:55.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.811038 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:32777: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:55.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.927610 1200 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:55.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.928191 1200 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:55.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.928820 1200 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:55.954589 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } attempt: 4
15:35:55.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.958350 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:32777: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.962348 1200 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:35:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.962572 1200 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:35:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.962831 1200 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:55.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.963040 1200 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:35:55.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.963310 1200 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:35:55.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.963519 1200 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:35:55.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:55.963662 1200 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36831/jwks.json
15:35:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:56.003405 1200 master_runner.cc:386] Master server non-default flags:
15:35:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:35:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:56.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.189.keytab
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:32777
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/info.pb
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/logs
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:56.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:56.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:56.004812 1200 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:56.006181 1200 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:56.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:56.026413 1208 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:56.257826 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } attempt: 5
15:35:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:56.261874 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:32777: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:56.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:56.809484 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } attempt: 6
15:35:56.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:56.817081 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.189:32777: Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:35:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:56.026957 1207 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:56.026420 1210 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.262217 1209 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:35:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.262351 1200 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:35:57.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.266600 1200 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.269901 1200 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:57.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.271394 1200 hybrid_clock.cc:648] HybridClock initialized: now 1790696157271362 us; error 52 us; skew 500 ppm
15:35:57.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.272215 1200 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:57.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:57 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696157, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:35:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.279088 1200 init.cc:377] Logged in from keytab as kudu/127.1.18.189@KRBTEST.COM (short username kudu)
15:35:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.285136 1200 webserver.cc:460] Webserver started at http://127.1.18.189:36979/ using document root <none> and password file <none>
15:35:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.286173 1200 fs_manager.cc:359] Metadata directory not provided
15:35:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.286371 1200 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:57.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.286800 1200 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.291682 1200 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/instance:
15:35:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0977f760b42746bb8e14f99f8ae57299"
15:35:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:57 on dist-test-slave-cq88"
15:35:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.292763 1200 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal/instance:
15:35:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0977f760b42746bb8e14f99f8ae57299"
15:35:57.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:57 on dist-test-slave-cq88"
15:35:57.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.300519 1200 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
15:35:57.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.306579 1220 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:57.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.307623 1200 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:35:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.307967 1200 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:35:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0977f760b42746bb8e14f99f8ae57299"
15:35:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:57 on dist-test-slave-cq88"
15:35:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.308310 1200 fs_report.cc:383] FS layout report
15:35:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:35:57.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/data
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:57.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:57.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.350304 1200 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:57.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.356595 1200 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:57.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.357149 1200 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.437731 1200 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.189:32777
15:35:57.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.437835 1271 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.189:32777 every 8 connection(s)
15:35:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.440429 1200 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/info.pb
15:35:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.442798 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 1200
15:35:57.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.443292 1098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal/instance
15:35:57.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.447089 1272 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:35:57.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.188@KRBTEST.COM; defaulting to no policy
15:35:57.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:35:57.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.188@KRBTEST.COM" created.
15:35:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.464193 1272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } has no permanent_uuid. Determining permanent_uuid...
15:35:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:57 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696157, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
15:35:57.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:35:57.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.188.keytab.
15:35:57.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.188.keytab.
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.500401 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/logs
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/info.pb
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:42991
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.188.keytab
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:35:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:35:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.503777 1272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } has no permanent_uuid. Determining permanent_uuid...
15:35:57.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:57 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696157, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
15:35:57.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.541008 1272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } has no permanent_uuid. Determining permanent_uuid...
15:35:57.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.543766 1223 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:57.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.547470 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:57.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.596908 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 1
15:35:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.600975 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:57.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.669392 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 2
15:35:57.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.672998 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:57.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.764370 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 3
15:35:57.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.768133 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:57.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.848791 1279 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:35:57.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.849327 1279 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:35:57.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.849895 1279 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:35:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.877671 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } attempt: 7
15:35:57.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.884300 1279 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:35:57.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.884577 1279 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:35:57.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.885028 1279 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:35:57.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.885349 1279 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:35:57.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.885663 1279 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:35:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.885999 1279 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:35:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.886240 1279 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36831/jwks.json
15:35:57.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:57 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696155, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
15:35:57.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.901378 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } has no permanent_uuid. Determining permanent_uuid...
15:35:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.906669 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:57.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.911559 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 4
15:35:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.915088 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.928707 1279 master_runner.cc:386] Master server non-default flags:
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.188.keytab
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:42991
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:35:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/info.pb
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/logs
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:35:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.929966 1279 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:35:57.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.931080 1279 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:35:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.947644 1293 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.965091 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 1
15:35:57.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.969120 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:58.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:58.050551 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 2
15:35:58.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:58.055064 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:58.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:58.140498 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 3
15:35:58.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:58.144459 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:58.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:58.214504 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 5
15:35:58.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:58.218525 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:58.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:58.285068 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 4
15:35:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:58.291378 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:58.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:58.575004 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 5
15:35:58.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:58.578917 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:58.766189 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 6
15:35:58.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:58.772917 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:59.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.131520 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } attempt: 6
15:35:59.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:59.138497 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.188:42991: Network error: Client connection negotiation failed: client connection to 127.1.18.188:42991: connect: Connection refused (error 111)
15:35:59.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.947657 1291 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:59.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:57.949366 1279 server_base.cc:1034] running on GCE node
15:35:59.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:57.947749 1290 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:35:59.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.195742 1279 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:35:59.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.198477 1279 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:35:59.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.199867 1279 hybrid_clock.cc:648] HybridClock initialized: now 1790696159199833 us; error 45 us; skew 500 ppm
15:35:59.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.200860 1279 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:35:59.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696159, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:35:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.208745 1279 init.cc:377] Logged in from keytab as kudu/127.1.18.188@KRBTEST.COM (short username kudu)
15:35:59.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.214754 1279 webserver.cc:460] Webserver started at http://127.1.18.188:38391/ using document root <none> and password file <none>
15:35:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.215803 1279 fs_manager.cc:359] Metadata directory not provided
15:35:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.216041 1279 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:35:59.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.216576 1279 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:35:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.221757 1279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/instance:
15:35:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "828449205df04f2caf6b0127f1584dcc"
15:35:59.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:59 on dist-test-slave-cq88"
15:35:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.222945 1279 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal/instance:
15:35:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "828449205df04f2caf6b0127f1584dcc"
15:35:59.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:59 on dist-test-slave-cq88"
15:35:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.231513 1279 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.002s sys 0.005s
15:35:59.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.237319 1306 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:35:59.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.238359 1279 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.238755 1279 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "828449205df04f2caf6b0127f1584dcc"
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:59 on dist-test-slave-cq88"
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.239101 1279 fs_report.cc:383] FS layout report
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/data
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:35:59.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:35:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:35:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:35:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:35:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:35:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:35:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:35:59.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:35:59.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.297449 1279 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:35:59.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.303759 1279 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:35:59.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.304263 1279 kserver.cc:163] Server-wide thread pool size limit: 3276
15:35:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.392809 1279 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.188:42991
15:35:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.392853 1357 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.188:42991 every 8 connection(s)
15:35:59.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.395673 1279 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/info.pb
15:35:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.398438 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 1279
15:35:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.398989 1098 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal/instance
15:35:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.401731 1358 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:35:59.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.420835 1358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } has no permanent_uuid. Determining permanent_uuid...
15:35:59.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696153, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.188@KRBTEST.COM
15:35:59.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696159, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
15:35:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.463515 1358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } has no permanent_uuid. Determining permanent_uuid...
15:35:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696159, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
15:35:59.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696153, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.189@KRBTEST.COM
15:35:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.493350 1358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } has no permanent_uuid. Determining permanent_uuid...
15:35:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696159, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for kudu/127.1.18.188@KRBTEST.COM
15:35:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696153, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.190@KRBTEST.COM
15:35:59.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.525735 1358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Bootstrap starting.
15:35:59.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.532130 1098 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
15:35:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.537989 1358 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Neither blocks nor log segments found. Creating new log.
15:35:59.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.540227 1358 log.cc:824] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Log is configured to *not* fsync() on all Append() calls
15:35:59.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:59.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:35:59.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.545881 1358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: No bootstrap required, opened a new log
15:35:59.555 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:35:59.556 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:35:59.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.18.190"
port: 33879
}
bound_http_address {
host: "127.1.18.190"
port: 43303
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.18.189"
port: 32777
}
bound_http_address {
host: "127.1.18.189"
port: 36979
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.18.188"
port: 42991
}
bound_http_address {
host: "127.1.18.188"
port: 38391
}
}
}
15:35:59.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:35:59.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:59.560 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:35:59.562 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:35:59.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.570027 1358 raft_consensus.cc:348] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:35:59.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.571094 1358 raft_consensus.cc:374] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:35:59.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.571452 1358 raft_consensus.cc:729] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 828449205df04f2caf6b0127f1584dcc, State: Initialized, Role: FOLLOWER
15:35:59.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.572928 1358 consensus_queue.cc:260] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:35:59.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.575979 1368 sys_catalog.cc:455] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } } }
15:35:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.577082 1368 sys_catalog.cc:458] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [sys.catalog]: This master's current role is: FOLLOWER
15:35:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.577832 1358 sys_catalog.cc:564] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [sys.catalog]: configured and running, proceeding with master startup.
15:35:59.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696159, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:35:59.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:35:59.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:59.602 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc
15:35:59.612 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:35:59.613 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc
15:35:59.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:59.618279 1380 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:35:59.618 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:35:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:35:59.618667 1380 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:35:59.624 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:35:59.628 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:35:59.628 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:35:59.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.604751 1279 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: OK, instance_id { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" instance_seqno: 1790696155489019 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:35:59.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.640947 1279 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:35:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:35:59 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696159, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:35:59.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:35:59.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:59.677 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc
15:35:59.679 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:35:59.680 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.188:42991
15:35:59.682 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:35:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.683449 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 1279
15:35:59.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:59.706 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.190:33879
15:35:59.706 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:35:59.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.706688 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 1124
15:35:59.730 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:59.731 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.189:32777
15:35:59.731 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:35:59.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.732191 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 1200
15:35:59.754 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:35:59.754 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.188:42991
15:35:59.755 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:35:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:35:59.757328 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:35:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:35:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data
15:35:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:35:59.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/logs
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/info.pb
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:42991
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38391
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.188.keytab
15:35:59.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
15:35:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:35:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:35:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:35:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:35:59.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:35:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:35:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:35:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:35:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:35:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:35:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:35:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:36:00.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.065331 1384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:36:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.065959 1384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:36:00.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.066902 1384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:36:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.101301 1384 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:36:00.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.101614 1384 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:36:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.101959 1384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:36:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.102169 1384 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:36:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.102347 1384 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:36:00.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.102533 1384 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:36:00.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.102813 1384 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36831/jwks.json
15:36:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:00.140872 1384 master_runner.cc:386] Master server non-default flags:
15:36:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:36:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:36:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:36:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:36:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data
15:36:00.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.188.keytab
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.188
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.188:42991
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/info.pb
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.188
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38391
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/logs
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:36:00.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:00.142019 1384 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:36:00.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:00.142992 1384 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:36:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.159559 1392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:01.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.160041 1393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:01.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:00.160185 1395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:01.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:00.162374 1384 server_base.cc:1034] running on GCE node
15:36:01.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.393761 1384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:36:01.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.396747 1384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:36:01.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.398157 1384 hybrid_clock.cc:648] HybridClock initialized: now 1790696161398138 us; error 65 us; skew 500 ppm
15:36:01.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.399139 1384 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:36:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:36:01 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696161, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:36:01.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.405350 1384 init.cc:377] Logged in from keytab as kudu/127.1.18.188@KRBTEST.COM (short username kudu)
15:36:01.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.411417 1384 webserver.cc:460] Webserver started at http://127.1.18.188:38391/ using document root <none> and password file <none>
15:36:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.412451 1384 fs_manager.cc:359] Metadata directory not provided
15:36:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.412714 1384 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:36:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.421687 1384 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.001s sys 0.004s
15:36:01.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.426362 1404 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:36:01.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.427656 1384 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:36:01.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.427970 1384 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "828449205df04f2caf6b0127f1584dcc"
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:59 on dist-test-slave-cq88"
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.430217 1384 fs_report.cc:383] FS layout report
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/wal
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/data
15:36:01.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:36:01.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:36:01.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.491382 1384 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:36:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.497906 1384 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:36:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.498322 1384 kserver.cc:163] Server-wide thread pool size limit: 3276
15:36:01.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.576411 1384 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.188:42991
15:36:01.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.576581 1455 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.188:42991 every 8 connection(s)
15:36:01.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.580338 1384 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-2/data/info.pb
15:36:01.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.586035 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 1384
15:36:01.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:36:01.586 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.190:33879
15:36:01.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:36:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.588294 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:36:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:36:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:36:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data
15:36:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:36:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/logs
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/info.pb
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:33879
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43303
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.190.keytab
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:36:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:36:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:36:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:36:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:36:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:36:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:36:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:36:01.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:36:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:36:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:36:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:36:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.591473 1456 sys_catalog.cc:263] Verifying existing consensus state
15:36:01.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.598681 1456 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Bootstrap starting.
15:36:01.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.644740 1456 log.cc:824] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Log is configured to *not* fsync() on all Append() calls
15:36:01.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.650090 1456 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:36:01.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.650887 1456 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Bootstrap complete.
15:36:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.675645 1456 raft_consensus.cc:348] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.678095 1456 raft_consensus.cc:374] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:36:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.678431 1456 raft_consensus.cc:729] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 828449205df04f2caf6b0127f1584dcc, State: Initialized, Role: FOLLOWER
15:36:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.679406 1456 consensus_queue.cc:260] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:01.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.681888 1460 sys_catalog.cc:455] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } } }
15:36:01.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.682757 1460 sys_catalog.cc:458] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [sys.catalog]: This master's current role is: FOLLOWER
15:36:01.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.683072 1456 sys_catalog.cc:564] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [sys.catalog]: configured and running, proceeding with master startup.
15:36:01.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.716979 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:01.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.717363 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:01.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.714768 1384 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111),
15:36:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:01.745695 1384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:36:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.754328 1410 tablet.cc:2369] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:36:01.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.945402 1458 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:36:01.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.946105 1458 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:36:01.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.946820 1458 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:36:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.994009 1458 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:36:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.994345 1458 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:36:01.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.994851 1458 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:36:01.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.995185 1458 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:36:01.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.995720 1458 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:36:01.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.996049 1458 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:36:01.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:01.996282 1458 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36831/jwks.json
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.062971 1458 master_runner.cc:386] Master server non-default flags:
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.190.keytab
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.190
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:36:02.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.190:33879
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/info.pb
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.190
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43303
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/logs
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:36:02.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:36:02.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.064566 1458 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:36:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.065992 1458 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:36:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.086584 1483 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:02.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.125094 1460 raft_consensus.cc:482] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:36:02.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.125766 1460 raft_consensus.cc:504] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.129635 1460 leader_election.cc:288] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879), 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777)
15:36:02.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.134250 1407 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111)
15:36:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.147740 1407 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879): Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111)
15:36:02.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.148963 1406 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777): Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:36:02.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.158887 1406 leader_election.cc:302] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 828449205df04f2caf6b0127f1584dcc; no voters: 026b7023b53e4564a8e30fceefb3255b, 0977f760b42746bb8e14f99f8ae57299
15:36:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.160118 1460 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:36:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.720999 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:02.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.721295 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:03.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:02.088675 1458 server_base.cc:1034] running on GCE node
15:36:03.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.090076 1480 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:02.086707 1481 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:03.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.390321 1458 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:36:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.393370 1458 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:36:03.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.394836 1458 hybrid_clock.cc:648] HybridClock initialized: now 1790696163394773 us; error 78 us; skew 500 ppm
15:36:03.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.395927 1458 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:36:03.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:36:03 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696163, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:36:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.408244 1458 init.cc:377] Logged in from keytab as kudu/127.1.18.190@KRBTEST.COM (short username kudu)
15:36:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.414132 1458 webserver.cc:460] Webserver started at http://127.1.18.190:43303/ using document root <none> and password file <none>
15:36:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.415019 1458 fs_manager.cc:359] Metadata directory not provided
15:36:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.415216 1458 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:36:03.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.423728 1458 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
15:36:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.428512 1494 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:36:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.429674 1458 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:36:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.430050 1458 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:36:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "026b7023b53e4564a8e30fceefb3255b"
15:36:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:55 on dist-test-slave-cq88"
15:36:03.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.432473 1458 fs_report.cc:383] FS layout report
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/data
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:36:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:36:03.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.483770 1458 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:36:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.491797 1458 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:36:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.492365 1458 kserver.cc:163] Server-wide thread pool size limit: 3276
15:36:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.584290 1458 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.190:33879
15:36:03.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.584367 1545 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.190:33879 every 8 connection(s)
15:36:03.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.590639 1458 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/data/info.pb
15:36:03.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.593474 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 1458
15:36:03.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:36:03.595 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.189:32777
15:36:03.596 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.601732 1098 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/logs
15:36:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/info.pb
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:32777
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36979
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.189.keytab
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:36:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:36:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:36:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.608368 1546 sys_catalog.cc:263] Verifying existing consensus state
15:36:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 15:36:03.609439 1546 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:36:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.610581 1458 master.cc:537] Master@127.1.18.190:33879 shutting down...
15:36:03.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:03.692512 1458 master.cc:552] Master@127.1.18.190:33879 shutdown complete.
15:36:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:36:03.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:03.724589 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:03.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:03.724896 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:03.976297 1548 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:36:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:03.976976 1548 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:36:03.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:03.977924 1548 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:36:04.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.010557 1548 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:36:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.010897 1548 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:36:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.011248 1548 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:36:04.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.011561 1548 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:36:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.011899 1548 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:36:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.012213 1548 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:36:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.012435 1548 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36831/jwks.json
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.050086 1548 master_runner.cc:386] Master server non-default flags:
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.148:34175
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.190:33879,127.1.18.189:32777,127.1.18.188:42991
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/krb5kdc/kudu_127.1.18.189.keytab
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.189
15:36:04.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.189:32777
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36831/jwks.json
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/info.pb
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.189
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36979
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/logs
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:36:04.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 14:50:29 UTC on ccycloud.quasar-iyffyt.root.comops.site
15:36:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4680449
15:36:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:36:04.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.051625 1548 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:36:04.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.052757 1548 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:36:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.101704 1560 raft_consensus.cc:482] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:36:04.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.102228 1560 raft_consensus.cc:504] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:04.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.104620 1560 leader_election.cc:288] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879), 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777)
15:36:04.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.112177 1407 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879): Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111)
15:36:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.112807 1406 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777): Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:36:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.113615 1406 leader_election.cc:302] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 828449205df04f2caf6b0127f1584dcc; no voters: 026b7023b53e4564a8e30fceefb3255b, 0977f760b42746bb8e14f99f8ae57299
15:36:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.114563 1560 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:36:04.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.729635 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:04.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.730305 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.071905 1554 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:05.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.072122 1557 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:04.073000 1548 server_base.cc:1034] running on GCE node
15:36:05.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:04.071902 1555 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:36:05.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.315769 1548 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:36:05.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.318815 1548 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:36:05.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.320302 1548 hybrid_clock.cc:648] HybridClock initialized: now 1790696165320266 us; error 53 us; skew 500 ppm
15:36:05.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.321180 1548 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:36:05.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:36:05 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696165, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:36:05.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.327450 1548 init.cc:377] Logged in from keytab as kudu/127.1.18.189@KRBTEST.COM (short username kudu)
15:36:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.332813 1548 webserver.cc:460] Webserver started at http://127.1.18.189:36979/ using document root <none> and password file <none>
15:36:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.333740 1548 fs_manager.cc:359] Metadata directory not provided
15:36:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.333935 1548 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:36:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.341666 1548 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
15:36:05.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.346171 1571 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:36:05.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.347405 1548 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.347823 1548 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data,/tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0977f760b42746bb8e14f99f8ae57299"
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 15:35:57 on dist-test-slave-cq88"
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.350150 1548 fs_report.cc:383] FS layout report
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal
15:36:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/data
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:36:05.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:36:05.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.408929 1548 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:36:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.415300 1548 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:36:05.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.415727 1548 kserver.cc:163] Server-wide thread pool size limit: 3276
15:36:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.493511 1548 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.189:32777
15:36:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.493683 1622 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.189:32777 every 8 connection(s)
15:36:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.496554 1548 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/data/info.pb
15:36:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.503626 1098 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu as pid 1548
15:36:05.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:36:05.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.508023 1623 sys_catalog.cc:263] Verifying existing consensus state
15:36:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 15:36:05.509763 1623 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:36:05.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.510226 1548 master.cc:537] Master@127.1.18.189:32777 shutting down...
15:36:05.565 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.188:42991(127.1.18.188:42991)] connecting to peer
15:36:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:05.581629 1548 master.cc:552] Master@127.1.18.189:32777 shutdown complete.
15:36:05.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskE66ITT/test-tmp/mini-kudu-cluster3202322973649375451/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:36:05.626 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:05.629 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:05.689 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.18.188:42991(127.1.18.188:42991)] Successfully connected to peer
15:36:05.691 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:05.692 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:05.694 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:05.734623 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:05.734949 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:05.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:05.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:05.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:05.770 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:36:05.778 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.18.188
15:36:05.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:36:05 dist-test-slave-cq88 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696159, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.188@KRBTEST.COM
15:36:05.986 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x4a4f5793, L:/127.0.0.1:33502 - R:/127.1.18.188:42991] using SASL/GSSAPI
15:36:06.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.011 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.015 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.015 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.015 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.019 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.019 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.020 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.020 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.029 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.031 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.032 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.033 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.034 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.035 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.035 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.036 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.037 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.049 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.054 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.056 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.056 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.056 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.057 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.069 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.071 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.074 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.075 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.075 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.075 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.089 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.090 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.092 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.093 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.094 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.095 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.095 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.096 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.129 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.131 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.132 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.132 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.133 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.134 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.134 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.136 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.136 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.249 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.251 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.252 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.252 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.254 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.255 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.256 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.256 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.257 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.389 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.391 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.392 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.392 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.393 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.394 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.394 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.395 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.395 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.490 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.491 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.492 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.492 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.494 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.494 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.495 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.495 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:06.738077 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:06.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:06.738417 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:06.748888 1384 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111),
15:36:06.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:06.749406 1384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:36:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:06.827665 1630 raft_consensus.cc:482] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:36:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:06.828027 1630 raft_consensus.cc:504] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:06.829677 1630 leader_election.cc:288] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879), 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777)
15:36:06.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:06.834278 1406 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777): Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:36:06.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:06.834779 1407 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879): Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111)
15:36:06.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:06.835103 1407 leader_election.cc:302] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 828449205df04f2caf6b0127f1584dcc; no voters: 026b7023b53e4564a8e30fceefb3255b, 0977f760b42746bb8e14f99f8ae57299
15:36:06.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:06.835705 1630 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:36:06.849 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.851 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.854 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.854 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.854 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:06.870 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:06.870 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:06.873 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.873 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:06.874 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.874 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.875 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:06.875 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:07.741931 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:07.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:07.742178 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:08.745364 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:08.745635 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:08.889 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:08.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:08.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:08.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:08.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:08.892 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:08.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:08.892 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:08.893 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:09.577854 1637 raft_consensus.cc:482] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:36:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:09.578199 1637 raft_consensus.cc:504] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:09.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:09.579805 1637 leader_election.cc:288] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879), 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777)
15:36:09.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:09.581647 1407 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111) [suppressed 5 similar messages]
15:36:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:09.584895 1407 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879): Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111)
15:36:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:09.585515 1406 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777): Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:36:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:09.585958 1406 leader_election.cc:302] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 828449205df04f2caf6b0127f1584dcc; no voters: 026b7023b53e4564a8e30fceefb3255b, 0977f760b42746bb8e14f99f8ae57299
15:36:09.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:09.586591 1637 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:36:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:09.748996 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:09.749403 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:10.753609 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:10.754031 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:36:11 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696171, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:36:11.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:11.409238 1400 init.cc:295] Successfully reacquired a new kerberos TGT
15:36:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:11.753105 1384 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111),
15:36:11.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:11.753675 1384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:36:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:11.757143 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:11.757436 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:11.849 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:11.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:11.853 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:11.854 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:11.854 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:11.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:11.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:11.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:11.855 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:12.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:12.760301 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:12.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:12.760563 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:13.009 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:13.011 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:13.012 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.012 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.014 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.014 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.014 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.014 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:13.049 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:13.050 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:13.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.052 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.052 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.054 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.054 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.054 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.055 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:13.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:13.171556 1647 raft_consensus.cc:482] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:36:13.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:13.172039 1647 raft_consensus.cc:504] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:13.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:13.173695 1647 leader_election.cc:288] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879), 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777)
15:36:13.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:13.178228 1407 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879): Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111)
15:36:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:13.178784 1406 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777): Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:36:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:13.179143 1406 leader_election.cc:302] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 828449205df04f2caf6b0127f1584dcc; no voters: 026b7023b53e4564a8e30fceefb3255b, 0977f760b42746bb8e14f99f8ae57299
15:36:13.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:13.179806 1647 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:36:13.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:13.764473 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:13.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:13.764796 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:13.889 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:13.890 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:13.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:13.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.894 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:13.894 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:14.140030 802 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9a00 after lost signal to thread 674
15:36:14.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:14.141044 802 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9e00 after lost signal to thread 805
15:36:14.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:14.211 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:14.211 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:14.211 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:14.211 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:14.213 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:14.213 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:14.213 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:14.214 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:14.768126 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:14.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:14.768427 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:15.441717 1452 debug-util.cc:398] Leaking SignalData structure 0x7b0800073aa0 after lost signal to thread 1386
15:36:15.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:15.442373 1452 debug-util.cc:398] Leaking SignalData structure 0x7b080008be80 after lost signal to thread 1455
15:36:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:15.772707 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:15.773411 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:16.776539 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:16.776829 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:16.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:16.805224 1384 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111),
15:36:16.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:16.805735 1384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:36:17.010 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:17.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:17.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:17.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:17.012 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:17.014 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:17.014 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:17.014 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:17.015 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:17.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:17.779567 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:17.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:17.779855 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:18.782631 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:18.782946 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:19.785991 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:19.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:19.786293 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:20.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:20.531 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:20.532 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:20.533 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:20.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:20.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:20.533 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:20.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:20.534 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:20.789541 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:20.789866 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:21.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:36:21 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696181, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:36:21.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:21.411814 1400 init.cc:295] Successfully reacquired a new kerberos TGT
15:36:21.649 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:21.650 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:21.650 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:21.651 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:21.651 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:21.652 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:21.652 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:21.652 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:21.653 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:21.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:21.792613 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:21.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:21.792874 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:21.809068 1384 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111),
15:36:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:21.809471 1384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:36:22.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:22.796293 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:22.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:22.796600 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:23.799443 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:23.799719 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:24.189834 1664 raft_consensus.cc:482] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:36:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:24.190299 1664 raft_consensus.cc:504] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "026b7023b53e4564a8e30fceefb3255b" member_type: VOTER last_known_addr { host: "127.1.18.190" port: 33879 } } peers { permanent_uuid: "0977f760b42746bb8e14f99f8ae57299" member_type: VOTER last_known_addr { host: "127.1.18.189" port: 32777 } } peers { permanent_uuid: "828449205df04f2caf6b0127f1584dcc" member_type: VOTER last_known_addr { host: "127.1.18.188" port: 42991 } }
15:36:24.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:24.192093 1664 leader_election.cc:288] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879), 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777)
15:36:24.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:24.194730 1407 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111) [suppressed 3 similar messages]
15:36:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:24.197712 1406 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0977f760b42746bb8e14f99f8ae57299 (127.1.18.189:32777): Network error: Client connection negotiation failed: client connection to 127.1.18.189:32777: connect: Connection refused (error 111)
15:36:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:24.198215 1407 leader_election.cc:334] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 026b7023b53e4564a8e30fceefb3255b (127.1.18.190:33879): Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111)
15:36:24.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:24.198482 1407 leader_election.cc:302] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 828449205df04f2caf6b0127f1584dcc; no voters: 026b7023b53e4564a8e30fceefb3255b, 0977f760b42746bb8e14f99f8ae57299
15:36:24.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:24.199066 1664 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:36:24.609 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:24.610 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:24.611 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:24.611 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:24.611 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:24.612 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:24.612 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:24.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:24.613 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:24.802419 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:24.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:24.802680 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:25.169 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:25.170 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:25.170 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:25.171 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:25.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:25.171 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:25.171 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:25.171 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:25.172 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:25.806165 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:25.806453 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:26.810 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:26.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:26.809669 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:26.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:26.810014 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:26.811 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:26.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:26.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:26.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:26.812 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:26.812 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:26.813 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:26.813684 1384 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111),
15:36:26.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:26.814097 1384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:36:26.815 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:27.813045 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:27.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:27.813380 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:28.816174 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:28.816468 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:29.689 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:29.690 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:29.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:29.690 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:29.690 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:29.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:29.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:29.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:29.693 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:29.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:29.819514 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:29.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:29.819811 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:30.749 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:30.751 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:30.751 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:30.751 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:30.752 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:30.752 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:30.752 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:30.752 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:30.753 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:30.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:30.822968 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:30.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:30.823339 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 15:36:31 dist-test-slave-cq88 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790696191, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:36:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:31.414188 1400 init.cc:295] Successfully reacquired a new kerberos TGT
15:36:31.569 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:31.570 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:31.570 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:31.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:31.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:31.571 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:31.571 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:31.571 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:31.573 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:31.817268 1384 master_runner.cc:186] Error getting master registration for 127.1.18.190:33879: Network error: Client connection negotiation failed: client connection to 127.1.18.190:33879: connect: Connection refused (error 111),
15:36:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:31.817728 1384 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:36:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:31.826283 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:31.826537 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:32.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:32.600230 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1600 after lost signal to thread 460
15:36:32.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:32.600782 525 debug-util.cc:398] Leaking SignalData structure 0x7b080009f620 after lost signal to thread 528
15:36:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:32.829198 1471 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 828449205df04f2caf6b0127f1584dcc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:36:32.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 15:36:32.829439 1471 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:36:33.189 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:33.191 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:33.191 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:33.191 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:33.191 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:33.192 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:33.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:33.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:33.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:33.250 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.190:33879(127.1.18.190:33879)] connecting to peer
15:36:33.251 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.189:32777(127.1.18.189:32777)] connecting to peer
15:36:33.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:33.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.190:33879(127.1.18.190:33879)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:33.251 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.190:33879: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879
15:36:33.252 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:33.252 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.189:32777(127.1.18.189:32777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:33.252 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.189:32777: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
15:36:33.256 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777; will retry
15:36:33.261 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27748), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [178ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [238ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [484ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [488ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [495ms] refreshing cache from master, [495ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [503ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [507ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [514ms] refreshing cache from master, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [523ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [534ms] refreshing cache from master, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [541ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [543ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [559ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [561ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [578ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [582ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [618ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [738ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [743ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [878ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [879ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [881ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [978ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [979ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [982ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1341ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1355ms] refreshing cache from master, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1361ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [3374ms] refreshing cache from master, [3374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [3375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [3376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [3378ms] too many traces: truncated at 100 traces, deferred=Deferred@194490187(state=PENDING, result=null, callback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726), errback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726)))
15:36:33.266 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.190:33879(127.1.18.190:33879)] disconnecting while in state TERMINATED
15:36:33.268 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.189:32777(127.1.18.189:32777)] disconnecting while in state TERMINATED
15:36:33.268 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.188:42991(127.1.18.188:42991)] disconnecting while in state READY
15:36:33.274 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:36:33.276 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.18.188:42991(127.1.18.188:42991)] handling channelInactive
15:36:33.276 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.188:42991(127.1.18.188:42991)] cleaning up while in state READY due to: connection closed
15:36:33.277 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:36:33.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:33.288961 1098 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 1384
15:36:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T15:36:34Z chronyd exiting
15:36:35.570 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:36:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:35.571836 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 654
15:36:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:35.597803 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 810
15:36:35.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:35.624081 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 957
15:36:35.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:35.649981 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 386
15:36:35.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:35.682813 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 456
15:36:35.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 15:36:35.720007 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskE66ITT/build/tsan/bin/kudu with pid 531
15:36:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T15:36:36Z chronyd exiting
15:36:37.907 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27748), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [178ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [238ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [484ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [488ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [495ms] refreshing cache from master, [495ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [503ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [507ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [514ms] refreshing cache from master, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [523ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [534ms] refreshing cache from master, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [541ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [543ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [559ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [561ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [578ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [582ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [618ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [738ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [743ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [878ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [879ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [881ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [978ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [979ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [982ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1341ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1355ms] refreshing cache from master, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1361ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [3374ms] refreshing cache from master, [3374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [3375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [3376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [3378ms] too many traces: truncated at 100 traces, deferred=Deferred@194490187(state=PENDING, result=null, callback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726), errback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:32777
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27748), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [178ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [238ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [484ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [488ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [495ms] refreshing cache from master, [495ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [503ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [507ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [514ms] refreshing cache from master, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [523ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [534ms] refreshing cache from master, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [541ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [543ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [559ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [561ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [578ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [582ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [618ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [738ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [743ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [878ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [879ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [881ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [978ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [979ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [982ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1341ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1355ms] refreshing cache from master, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1361ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [3374ms] refreshing cache from master, [3374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [3375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [3376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [3378ms] too many traces: truncated at 100 traces, deferred=Deferred@194490187(state=PENDING, result=null, callback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726), errback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:32777
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
15:36:39.096 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
15:36:39.096 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27748), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [178ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [238ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [484ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [488ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [495ms] refreshing cache from master, [495ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [503ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [507ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [514ms] refreshing cache from master, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [523ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [534ms] refreshing cache from master, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [541ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [543ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [559ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [561ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [578ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [582ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [618ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [738ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [743ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [878ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [879ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [881ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [978ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [979ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [982ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1341ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1355ms] refreshing cache from master, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1361ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [3374ms] refreshing cache from master, [3374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [3375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [3376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [3378ms] too many traces: truncated at 100 traces, deferred=Deferred@194490187(state=PENDING, result=null, callback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726), errback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:32777
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=27748), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [115ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [178ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [238ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [484ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [488ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [495ms] refreshing cache from master, [495ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [497ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [501ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [501ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [503ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [505ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [507ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [514ms] refreshing cache from master, [514ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [519ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [520ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [522ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [523ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [534ms] refreshing cache from master, [534ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [541ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [542ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [543ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [554ms] refreshing cache from master, [554ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [557ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [558ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [559ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [561ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [561ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [574ms] refreshing cache from master, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [574ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [578ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [580ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [581ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [582ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [614ms] refreshing cache from master, [614ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [618ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [738ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [739ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [742ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [743ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [874ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [878ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [879ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [880ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [881ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [978ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [979ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [982ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1335ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1341ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1355ms] refreshing cache from master, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [1355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.190:33879: Network error: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.188:42991: OK, [1360ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.189:32777: Network error: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [1361ms] delaying RPC due to: Service unavailable: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777, [3374ms] refreshing cache from master, [3374ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.188:42991, [3375ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.190:33879, [3376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.189:32777, [3378ms] too many traces: truncated at 100 traces, deferred=Deferred@194490187(state=PENDING, result=null, callback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726), errback=(continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.188:42991,127.1.18.190:33879,127.1.18.189:32777) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.190:33879(127.1.18.190:33879): Connection refused: /127.1.18.190:33879,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.189:32777(127.1.18.189:32777): Connection refused: /127.1.18.189:32777
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.189:32777
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)