11:10:50.263 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
11:10:50.279 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskV256yr/test-tmp
11:10:50.282 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:10:50.288 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskV256yr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:10:50.690 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
11:10:51.023 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:10:51.024 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:10:51.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.029713 20530 mini_oidc.cc:152] Starting JWKS server
11:10:51.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.041004 20530 webserver.cc:460] Webserver started at http://127.0.0.1:33829/ using document root <none> and password file <none>
11:10:51.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.041668 20530 mini_oidc.cc:189] Starting OIDC Discovery server
11:10:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.048164 20530 webserver.cc:460] Webserver started at http://127.0.0.1:33933/ using document root <none> and password file <none>
11:10:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T11:10:51Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:10:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T11:10:51Z Disabled control of system clock
11:10:51.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.127223 20530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/wal
11:10:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data
11:10:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:10:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:10:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:10:51.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/logs
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data/info.pb
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:10:51.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.190:43375
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.190
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:10:51.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:10:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:10:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:10:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:10:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:10:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:10:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:10:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33829/jwks.json with env {}
11:10:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.719635 20543 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:10:51.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.720319 20543 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:10:51.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.721021 20543 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:10:51.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.756105 20543 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:10:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.756408 20543 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:10:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.756703 20543 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:10:51.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.757105 20543 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:10:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.757444 20543 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:10:51.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.757714 20543 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33829/jwks.json
11:10:51.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.795825 20543 master_runner.cc:386] Master server non-default flags:
11:10:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:10:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:10:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:10:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:10:51.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/wal
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.190:43375
11:10:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33829/jwks.json
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data/info.pb
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.190
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:10:51.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/logs
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:10:51.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:10:51.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:10:51.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.797870 20543 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:10:51.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:51.800414 20543 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:10:51.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.856109 20550 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.858788 20549 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:53.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:51.862246 20553 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:53.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.317108 20543 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:10:53.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:53.317988 20552 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:10:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.321934 20543 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:10:53.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.325721 20543 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:10:53.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.327112 20543 hybrid_clock.cc:648] HybridClock initialized: now 1786101053327063 us; error 70 us; skew 500 ppm
11:10:53.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.328261 20543 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:10:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.347913 20543 webserver.cc:460] Webserver started at http://127.20.12.190:39361/ using document root <none> and password file <none>
11:10:53.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.349462 20543 fs_manager.cc:359] Metadata directory not provided
11:10:53.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.349687 20543 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:10:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.350258 20543 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:10:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.357419 20543 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data/instance:
11:10:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5047ca60beab474aa1c4fa171befd914"
11:10:53.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:53 on dist-test-slave-77v9"
11:10:53.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.359210 20543 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/wal/instance:
11:10:53.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5047ca60beab474aa1c4fa171befd914"
11:10:53.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:53 on dist-test-slave-77v9"
11:10:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.371273 20543 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.008s sys 0.004s
11:10:53.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.383260 20560 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:10:53.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.385583 20543 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.004s sys 0.006s
11:10:53.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.386045 20543 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/wal
11:10:53.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5047ca60beab474aa1c4fa171befd914"
11:10:53.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:53 on dist-test-slave-77v9"
11:10:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.386428 20543 fs_report.cc:383] FS layout report
11:10:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:10:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/wal
11:10:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/wal
11:10:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data/data
11:10:53.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:10:53.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:10:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.459334 20543 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:10:53.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.461818 20543 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:10:53.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.462388 20543 kserver.cc:163] Server-wide thread pool size limit: 3276
11:10:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.681448 20543 rpc_server.cc:307] RPC server started. Bound to: 127.20.12.190:43375
11:10:53.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.681649 20611 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.12.190:43375 every 8 connection(s)
11:10:53.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.684949 20543 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/data/info.pb
11:10:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.693578 20530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 20543
11:10:53.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.694283 20530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-0/wal/instance
11:10:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.701870 20612 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:10:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.702248 20530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/wal
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:10:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/logs
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data/info.pb
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:10:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.189:37605
11:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.189
11:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:10:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:10:53.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:10:53.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:10:53.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:10:53.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33829/jwks.json with env {}
11:10:53.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.735914 20612 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } has no permanent_uuid. Determining permanent_uuid...
11:10:53.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.797240 20612 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } has no permanent_uuid. Determining permanent_uuid...
11:10:53.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:53.801393 20563 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:53.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:53.811483 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:53.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.861213 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 1
11:10:53.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:53.866317 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:53.934790 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 2
11:10:53.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:53.940436 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:54.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:54.032169 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 3
11:10:54.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.039124 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:54.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:54.182814 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 4
11:10:54.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.187690 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:54.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.262171 20614 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:10:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.262854 20614 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:10:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.263326 20614 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:10:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.297127 20614 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:10:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.297420 20614 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:10:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.297613 20614 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.297884 20614 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.298074 20614 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:10:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.298203 20614 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33829/jwks.json
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:54.335551 20614 master_runner.cc:386] Master server non-default flags:
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/wal
11:10:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.189:37605
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33829/jwks.json
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:10:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data/info.pb
11:10:54.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.189
11:10:54.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/logs
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:10:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:54.336997 20614 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:10:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:54.338824 20614 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:10:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.378680 20624 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:54.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:54.487349 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 5
11:10:54.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.496152 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.044102 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 6
11:10:55.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:55.067036 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:55.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.379141 20626 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:55.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:54.382892 20623 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:55.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:55.878378 20625 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1499 milliseconds
11:10:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.878515 20614 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:10:55.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.880010 20614 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:10:55.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.884385 20614 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:10:55.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.885804 20614 hybrid_clock.cc:648] HybridClock initialized: now 1786101055885755 us; error 68 us; skew 500 ppm
11:10:55.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.886953 20614 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:10:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.901662 20614 webserver.cc:460] Webserver started at http://127.20.12.189:35113/ using document root <none> and password file <none>
11:10:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.902676 20614 fs_manager.cc:359] Metadata directory not provided
11:10:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.902910 20614 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:10:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.903312 20614 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:10:55.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.909134 20614 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data/instance:
11:10:55.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b2439139c434a0b98e5962dddb77189"
11:10:55.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:55 on dist-test-slave-77v9"
11:10:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.910604 20614 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/wal/instance:
11:10:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b2439139c434a0b98e5962dddb77189"
11:10:55.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:55 on dist-test-slave-77v9"
11:10:55.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.923243 20614 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.010s sys 0.002s
11:10:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.935923 20635 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:10:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.937390 20614 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.004s sys 0.002s
11:10:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.937906 20614 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/wal
11:10:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b2439139c434a0b98e5962dddb77189"
11:10:55.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:55 on dist-test-slave-77v9"
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:55.938490 20614 fs_report.cc:383] FS layout report
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/wal
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/wal
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data/data
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:10:55.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:10:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:10:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:10:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:10:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:10:55.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:10:56.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.035379 20614 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:10:56.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.038713 20614 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:10:56.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.039603 20614 kserver.cc:163] Server-wide thread pool size limit: 3276
11:10:56.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.127802 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 7
11:10:56.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:56.154060 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.189:37605: Network error: Client connection negotiation failed: client connection to 127.20.12.189:37605: connect: Connection refused (error 111)
11:10:56.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.365022 20614 rpc_server.cc:307] RPC server started. Bound to: 127.20.12.189:37605
11:10:56.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.365478 20688 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.12.189:37605 every 8 connection(s)
11:10:56.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.368248 20614 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/data/info.pb
11:10:56.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.370967 20530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 20614
11:10:56.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.371591 20530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-1/wal/instance
11:10:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.375936 20530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/wal
11:10:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data
11:10:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:10:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:10:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:10:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:10:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/logs
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data/info.pb
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:10:56.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:10:56.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:10:56.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.188:45025
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.188
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:10:56.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:10:56.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:10:56.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33829/jwks.json with env {}
11:10:56.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.388489 20689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:10:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.425907 20689 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } has no permanent_uuid. Determining permanent_uuid...
11:10:56.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.477808 20689 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } has no permanent_uuid. Determining permanent_uuid...
11:10:56.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.514333 20689 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } has no permanent_uuid. Determining permanent_uuid...
11:10:56.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:56.518772 20639 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:56.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:56.534471 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:56.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.584148 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 1
11:10:56.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:56.592506 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:56.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.661108 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 2
11:10:56.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:56.667232 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.758651 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 3
11:10:56.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:56.765147 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:56.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:56.908891 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 4
11:10:56.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:56.914865 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:57.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.002413 20691 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:10:57.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.003249 20691 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:10:57.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.003839 20691 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:10:57.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.039130 20691 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:10:57.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.039429 20691 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:10:57.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.039633 20691 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:10:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.039819 20691 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:10:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.040014 20691 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:10:57.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.040144 20691 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33829/jwks.json
11:10:57.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:57.076128 20691 master_runner.cc:386] Master server non-default flags:
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/wal
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.188:45025
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33829/jwks.json
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data/info.pb
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.188
11:10:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:10:57.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:10:57.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/logs
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:10:57.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:10:57.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:57.077637 20691 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:10:57.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:57.079818 20691 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:10:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.129200 20702 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:57.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:57.214324 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 5
11:10:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.220479 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:57.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:57.768122 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 6
11:10:57.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.785364 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.244758 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } attempt: 8
11:10:58.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.265002 20612 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } has no permanent_uuid. Determining permanent_uuid...
11:10:58.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:58.274197 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:58.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.339607 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 1
11:10:58.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:58.344213 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.397845 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 2
11:10:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:58.402920 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:58.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.479597 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 3
11:10:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:58.486317 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.129508 20701 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:58.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:57.132145 20704 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:10:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:58.532034 20703 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:10:58.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.532248 20691 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:10:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.536543 20691 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:10:58.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.540447 20691 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:10:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.541800 20691 hybrid_clock.cc:648] HybridClock initialized: now 1786101058541768 us; error 54 us; skew 500 ppm
11:10:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.542742 20691 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:10:58.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.561002 20691 webserver.cc:460] Webserver started at http://127.20.12.188:44359/ using document root <none> and password file <none>
11:10:58.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.562165 20691 fs_manager.cc:359] Metadata directory not provided
11:10:58.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.562358 20691 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:10:58.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.562969 20691 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:10:58.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.569320 20691 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data/instance:
11:10:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9"
11:10:58.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:58 on dist-test-slave-77v9"
11:10:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.570933 20691 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/wal/instance:
11:10:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9"
11:10:58.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:58 on dist-test-slave-77v9"
11:10:58.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.582628 20691 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.002s sys 0.010s
11:10:58.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.591540 20716 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:10:58.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.593357 20691 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.002s sys 0.005s
11:10:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.593809 20691 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/wal
11:10:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9"
11:10:58.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:10:58 on dist-test-slave-77v9"
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.594354 20691 fs_report.cc:383] FS layout report
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/wal
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/wal
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data/data
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:10:58.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:10:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:10:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:10:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:10:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:10:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:10:58.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:10:58.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.641898 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 4
11:10:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.644889 20691 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:10:58.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.647156 20691 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:10:58.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:58.647394 20612 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:58.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.648890 20691 kserver.cc:163] Server-wide thread pool size limit: 3276
11:10:58.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.846122 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 7
11:10:58.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:58.865264 20689 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.12.188:45025: Network error: Client connection negotiation failed: client connection to 127.20.12.188:45025: connect: Connection refused (error 111)
11:10:58.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.890236 20691 rpc_server.cc:307] RPC server started. Bound to: 127.20.12.188:45025
11:10:58.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.890645 20770 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.12.188:45025 every 8 connection(s)
11:10:58.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.893357 20691 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/data/info.pb
11:10:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.897001 20530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 20691
11:10:58.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.897603 20530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/master-2/wal/instance
11:10:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.904963 20771 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:10:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.943846 20612 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 5
11:10:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:58.946700 20771 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } has no permanent_uuid. Determining permanent_uuid...
11:10:59.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.020484 20771 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } has no permanent_uuid. Determining permanent_uuid...
11:10:59.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.026201 20612 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: Bootstrap starting.
11:10:59.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.047727 20771 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } has no permanent_uuid. Determining permanent_uuid...
11:10:59.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.048882 20612 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: Neither blocks nor log segments found. Creating new log.
11:10:59.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.055275 20612 log.cc:824] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: Log is configured to *not* fsync() on all Append() calls
11:10:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.070627 20612 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: No bootstrap required, opened a new log
11:10:59.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.082661 20771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9: Bootstrap starting.
11:10:59.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.097137 20771 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9: Neither blocks nor log segments found. Creating new log.
11:10:59.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.101899 20771 log.cc:824] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9: Log is configured to *not* fsync() on all Append() calls
11:10:59.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.112277 20771 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9: No bootstrap required, opened a new log
11:10:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.116441 20612 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.117466 20612 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:10:59.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.117897 20612 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5047ca60beab474aa1c4fa171befd914, State: Initialized, Role: FOLLOWER
11:10:59.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.119308 20612 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.130631 20782 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:10:59.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.132288 20782 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [sys.catalog]: This master's current role is: FOLLOWER
11:10:59.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.133623 20612 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [sys.catalog]: configured and running, proceeding with master startup.
11:10:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.169857 20771 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.172012 20771 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.172936 20771 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2efeb1f679f54aa1b3a1d7c5c67f8ed9, State: Initialized, Role: FOLLOWER
11:10:59.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.175067 20771 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.186058 20788 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:10:59.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.190362 20771 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: configured and running, proceeding with master startup.
11:10:59.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.188970 20788 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: This master's current role is: FOLLOWER
11:10:59.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:59.233708 20566 tablet.cc:2369] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:10:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.240926 20782 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:10:59.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.246364 20782 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.251466 20788 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:10:59.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.252641 20788 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.263051 20587 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5047ca60beab474aa1c4fa171befd914" is_pre_election: true
11:10:59.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.264954 20587 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2efeb1f679f54aa1b3a1d7c5c67f8ed9 in term 0.
11:10:59.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.267097 20788 leader_election.cc:288] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5047ca60beab474aa1c4fa171befd914 (127.20.12.190:43375), 2b2439139c434a0b98e5962dddb77189 (127.20.12.189:37605)
11:10:59.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.269045 20720 leader_election.cc:302] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2efeb1f679f54aa1b3a1d7c5c67f8ed9, 5047ca60beab474aa1c4fa171befd914; no voters:
11:10:59.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.271344 20788 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Leader pre-election won for term 1
11:10:59.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.272224 20788 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:10:59.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.274159 20788 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 0 FOLLOWER]: Advancing to term 1
11:10:59.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:59.282260 20797 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:10:59.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:59.283164 20797 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:10:59.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.294620 20788 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.303016 20788 leader_election.cc:288] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [CANDIDATE]: Term 1 election: Requested vote from peers 5047ca60beab474aa1c4fa171befd914 (127.20.12.190:43375), 2b2439139c434a0b98e5962dddb77189 (127.20.12.189:37605)
11:10:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.305124 20587 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5047ca60beab474aa1c4fa171befd914"
11:10:59.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.306180 20587 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 0 FOLLOWER]: Advancing to term 1
11:10:59.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.308794 20782 leader_election.cc:288] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2b2439139c434a0b98e5962dddb77189 (127.20.12.189:37605), 2efeb1f679f54aa1b3a1d7c5c67f8ed9 (127.20.12.188:45025)
11:10:59.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.312224 20744 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5047ca60beab474aa1c4fa171befd914" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" is_pre_election: true
11:10:59.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.314422 20744 raft_consensus.cc:2379] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 5047ca60beab474aa1c4fa171befd914 in current term 1: Already voted for candidate 2efeb1f679f54aa1b3a1d7c5c67f8ed9 in this term.
11:10:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.326468 20587 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2efeb1f679f54aa1b3a1d7c5c67f8ed9 in term 1.
11:10:59.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.328576 20720 leader_election.cc:302] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2efeb1f679f54aa1b3a1d7c5c67f8ed9, 5047ca60beab474aa1c4fa171befd914; no voters:
11:10:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.330309 20788 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 1 FOLLOWER]: Leader election won for term 1
11:10:59.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.342953 20788 raft_consensus.cc:686] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [term 1 LEADER]: Becoming Leader. State: Replica: 2efeb1f679f54aa1b3a1d7c5c67f8ed9, State: Running, Role: LEADER
11:10:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.345309 20788 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:10:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.240284 20543 master_runner.cc:186] Error getting master registration for 127.20.12.189:37605: OK, instance_id { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" instance_seqno: 1786101056271170 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:10:59.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.363976 20543 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:10:59.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.379842 20806 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2efeb1f679f54aa1b3a1d7c5c67f8ed9. Latest consensus state: current_term: 1 leader_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:10:59.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.381845 20806 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: This master's current role is: LEADER
11:10:59.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.408061 20809 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:10:59.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:10:59.412854 20723 tablet.cc:2369] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:10:59.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.475422 20809 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:10:59.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.439819 20691 master_runner.cc:186] Error getting master registration for 127.20.12.189:37605: OK, instance_id { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" instance_seqno: 1786101056271170 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:10:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.518438 20691 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:10:59.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.552310 20587 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 1 FOLLOWER]: Refusing update from remote peer 2efeb1f679f54aa1b3a1d7c5c67f8ed9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:10:59.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.555037 20806 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
11:10:59.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.597337 20782 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2efeb1f679f54aa1b3a1d7c5c67f8ed9. Latest consensus state: current_term: 1 leader_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:10:59.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.605613 20782 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [sys.catalog]: This master's current role is: FOLLOWER
11:10:59.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.613893 20811 mvcc.cc:205] Tried to move back new op lower bound from 7315869939880960000 to 7315869939096956928. Current Snapshot: MvccSnapshot[applied={T|T < 7315869939880960000}]
11:10:59.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.625530 20788 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:10:59.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.626902 20788 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: This master's current role is: LEADER
11:10:59.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.645066 20782 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:10:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.655735 20809 catalog_manager.cc:1347] Generated new cluster ID: e74c0f8713bc42e9bf9c6a892d216536
11:10:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.656281 20809 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:10:59.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.655622 20782 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [sys.catalog]: This master's current role is: FOLLOWER
11:10:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.766387 20809 catalog_manager.cc:1370] Generated new certificate authority record
11:10:59.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.776453 20809 catalog_manager.cc:1504] Loading token signing keys...
11:10:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.853538 20809 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9: Generated new TSK 0
11:10:59.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.856106 20809 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:10:59.909499 20530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/wal
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:10:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/logs
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data/info.pb
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.129:0
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.12.129
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.129
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:10:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:11:00.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:00.293627 20797 catalog_manager.cc:1259] Loaded cluster ID: e74c0f8713bc42e9bf9c6a892d216536
11:11:00.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:00.304939 20797 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: loading cluster ID for follower catalog manager: success
11:11:00.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:00.320441 20797 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: acquiring CA information for follower catalog manager: success
11:11:00.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:00.344120 20797 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:11:00.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:00.870002 20563 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.20.12.189:37605 timed out after 1.562s (SENT) [suppressed 12 similar messages]
11:11:00.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:00.882310 20720 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.20.12.189:37605 timed out after 1.616s (SENT)
11:11:00.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:00.930639 20817 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:00.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:00.931540 20817 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:00.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:00.932466 20817 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:00.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:00.955812 20689 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } attempt: 8
11:11:01.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.010794 20817 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:01.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.012679 20817 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.12.129
11:11:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.047444 20689 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: Bootstrap starting.
11:11:01.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.068799 20689 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: Neither blocks nor log segments found. Creating new log.
11:11:01.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.071653 20689 log.cc:824] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: Log is configured to *not* fsync() on all Append() calls
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.085844 20817 tablet_server_runner.cc:78] Tablet server non-default flags:
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/wal
11:11:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.129:0
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data/info.pb
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.129
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.12.129
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/logs
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:01.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:01.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.087720 20817 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:01.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.088338 20689 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: No bootstrap required, opened a new log
11:11:01.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.090301 20817 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:01.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.174439 20832 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.191550 20689 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.205845 20689 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:11:01.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.207472 20689 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2b2439139c434a0b98e5962dddb77189, State: Initialized, Role: FOLLOWER
11:11:01.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.209237 20689 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } }
11:11:01.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.258929 20659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5047ca60beab474aa1c4fa171befd914" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b2439139c434a0b98e5962dddb77189" is_pre_election: true
11:11:01.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.256384 20689 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [sys.catalog]: configured and running, proceeding with master startup.
11:11:01.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.266511 20563 leader_election.cc:302] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5047ca60beab474aa1c4fa171befd914; no voters: 2b2439139c434a0b98e5962dddb77189, 2efeb1f679f54aa1b3a1d7c5c67f8ed9
11:11:01.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.265079 20660 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b2439139c434a0b98e5962dddb77189" is_pre_election: true
11:11:01.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.251466 20836 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:11:01.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.261590 20663 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b2439139c434a0b98e5962dddb77189"
11:11:01.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.270989 20664 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b2439139c434a0b98e5962dddb77189" is_pre_election: true
11:11:01.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.276796 20663 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54304 (request call id 2) took 1972 ms (client timeout 2086 ms). Trace:
11:11:01.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.274003 20836 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [sys.catalog]: This master's current role is: FOLLOWER
11:11:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.260597 20661 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [term 0 FOLLOWER]: Advancing to term 1
11:11:01.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.278764 20662 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "5047ca60beab474aa1c4fa171befd914" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b2439139c434a0b98e5962dddb77189" is_pre_election: true
11:11:01.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.279820 20663 rpcz_store.cc:269] 0807 11:10:59.304153 (+ 0us) service_pool.cc:170] Inserting onto call queue
11:11:01.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:10:59.308400 (+ 4247us) service_pool.cc:227] Handling call
11:11:01.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:11:01.276619 (+1968219us) inbound_call.cc:173] Queueing success response
11:11:01.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
11:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.284623 20664 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54304 (request call id 1) took 2019 ms (client timeout 1616 ms). Trace:
11:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.288478 20664 rpcz_store.cc:269] 0807 11:10:59.265688 (+ 0us) service_pool.cc:170] Inserting onto call queue
11:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:10:59.265987 (+ 299us) service_pool.cc:227] Handling call
11:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:11:01.284492 (+2018505us) inbound_call.cc:173] Queueing success response
11:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
11:11:01.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.289105 20662 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:41764 (request call id 2) took 1979 ms (client timeout 1562 ms). Trace:
11:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.290316 20662 rpcz_store.cc:269] 0807 11:10:59.308519 (+ 0us) service_pool.cc:170] Inserting onto call queue
11:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:10:59.308967 (+ 448us) service_pool.cc:227] Handling call
11:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:11:01.287245 (+1978278us) inbound_call.cc:173] Queueing success response
11:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
11:11:01.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.281824 20838 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 5047ca60beab474aa1c4fa171befd914 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:11:01.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.302609 20661 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [term 1 FOLLOWER]: Refusing update from remote peer 2efeb1f679f54aa1b3a1d7c5c67f8ed9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:11:01.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.303836 20661 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:54304 (request call id 4) took 1757 ms. Trace:
11:11:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.304348 20661 rpcz_store.cc:276] 0807 11:10:59.546370 (+ 0us) service_pool.cc:170] Inserting onto call queue
11:11:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:10:59.546531 (+ 161us) service_pool.cc:227] Handling call
11:11:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:11:01.258786 (+1712255us) raft_consensus.cc:1457] Updating replica for 0 ops
11:11:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:11:01.303251 (+ 44465us) raft_consensus.cc:1683] Filling consensus response to leader.
11:11:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0807 11:11:01.303742 (+ 491us) inbound_call.cc:173] Queueing success response
11:11:01.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
11:11:01.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.307261 20818 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
11:11:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.313937 20641 tablet.cc:2369] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:11:01.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.512240 20836 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2efeb1f679f54aa1b3a1d7c5c67f8ed9. Latest consensus state: current_term: 1 leader_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:11:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.511262 20847 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:11:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.515959 20847 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [sys.catalog]: This master's current role is: FOLLOWER
11:11:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.513662 20836 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189 [sys.catalog]: This master's current role is: FOLLOWER
11:11:01.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.569753 20856 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:11:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.757411 20818 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5047ca60beab474aa1c4fa171befd914" member_type: VOTER last_known_addr { host: "127.20.12.190" port: 43375 } } peers { permanent_uuid: "2b2439139c434a0b98e5962dddb77189" member_type: VOTER last_known_addr { host: "127.20.12.189" port: 37605 } } peers { permanent_uuid: "2efeb1f679f54aa1b3a1d7c5c67f8ed9" member_type: VOTER last_known_addr { host: "127.20.12.188" port: 45025 } } }
11:11:01.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:01.758549 20818 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2efeb1f679f54aa1b3a1d7c5c67f8ed9 [sys.catalog]: This master's current role is: LEADER
11:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:02.592736 20856 catalog_manager.cc:1259] Loaded cluster ID: e74c0f8713bc42e9bf9c6a892d216536
11:11:02.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:02.593185 20856 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: loading cluster ID for follower catalog manager: success
11:11:02.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:02.622624 20856 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: acquiring CA information for follower catalog manager: success
11:11:02.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:02.649806 20856 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2b2439139c434a0b98e5962dddb77189: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:11:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.191545 20831 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:01.203044 20835 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:03.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:03.865377 20833 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2684 milliseconds
11:11:03.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.866110 20817 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:11:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.867846 20817 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:03.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.873063 20817 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.875264 20817 hybrid_clock.cc:648] HybridClock initialized: now 1786101063875109 us; error 179 us; skew 500 ppm
11:11:03.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.876694 20817 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:03.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.909526 20817 webserver.cc:460] Webserver started at http://127.20.12.129:42675/ using document root <none> and password file <none>
11:11:03.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.911005 20817 fs_manager.cc:359] Metadata directory not provided
11:11:03.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.911269 20817 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:03.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.911832 20817 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:11:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.920656 20817 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data/instance:
11:11:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d835fb0e4f4446b3be526ebd00a86565"
11:11:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:03 on dist-test-slave-77v9"
11:11:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.922326 20817 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/wal/instance:
11:11:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d835fb0e4f4446b3be526ebd00a86565"
11:11:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:03 on dist-test-slave-77v9"
11:11:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.936566 20817 fs_manager.cc:691] Time spent creating directory manager: real 0.013s user 0.001s sys 0.012s
11:11:03.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.948801 20867 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.950762 20817 fs_manager.cc:725] Time spent opening block manager: real 0.009s user 0.002s sys 0.005s
11:11:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.951170 20817 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/wal
11:11:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d835fb0e4f4446b3be526ebd00a86565"
11:11:03.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:03 on dist-test-slave-77v9"
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:03.951645 20817 fs_report.cc:383] FS layout report
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/wal
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/wal
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data/data
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:03.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:03.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:03.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:03.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:03.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:04.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.063371 20817 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:04.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.066128 20817 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:04.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.067031 20817 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:04.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.075269 20817 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:11:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.086660 20817 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:11:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.087026 20817 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:11:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.087427 20817 ts_tablet_manager.cc:610] Registered 0 tablets
11:11:04.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.087671 20817 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:11:04.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.608355 20817 rpc_server.cc:307] RPC server started. Bound to: 127.20.12.129:35605
11:11:04.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.609367 20984 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.12.129:35605 every 8 connection(s)
11:11:04.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.612777 20817 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/data/info.pb
11:11:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.622822 20530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 20817
11:11:04.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.624363 20530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-0/wal/instance
11:11:04.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.686057 20530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/wal
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:04.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/logs
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data/info.pb
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.130:0
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.12.130
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.130
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:04.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:11:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.768499 20989 heartbeater.cc:347] Connected to a master server at 127.20.12.190:43375
11:11:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.769737 20989 heartbeater.cc:464] Registering TS with master...
11:11:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.771070 20985 heartbeater.cc:347] Connected to a master server at 127.20.12.188:45025
11:11:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.771656 20985 heartbeater.cc:464] Registering TS with master...
11:11:04.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.773788 20986 heartbeater.cc:347] Connected to a master server at 127.20.12.189:37605
11:11:04.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.774262 20989 heartbeater.cc:511] Master 127.20.12.190:43375 requested a full tablet report, sending...
11:11:04.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.774569 20986 heartbeater.cc:464] Registering TS with master...
11:11:04.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.776626 20986 heartbeater.cc:511] Master 127.20.12.189:37605 requested a full tablet report, sending...
11:11:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.777480 20985 heartbeater.cc:511] Master 127.20.12.188:45025 requested a full tablet report, sending...
11:11:04.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.784843 20652 ts_manager.cc:194] Registered new tserver with Master: d835fb0e4f4446b3be526ebd00a86565 (127.20.12.129:35605)
11:11:04.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.783928 20576 ts_manager.cc:194] Registered new tserver with Master: d835fb0e4f4446b3be526ebd00a86565 (127.20.12.129:35605)
11:11:04.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.790117 20734 ts_manager.cc:194] Registered new tserver with Master: d835fb0e4f4446b3be526ebd00a86565 (127.20.12.129:35605)
11:11:04.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:04.797613 20734 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.12.129:35397
11:11:05.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:04.999534 20767 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5b20 after lost signal to thread 20696
11:11:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.594339 20995 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.595077 20995 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.595923 20995 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:05.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.669133 20995 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.670857 20995 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.12.130
11:11:05.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:05.748225 20995 tablet_server_runner.cc:78] Tablet server non-default flags:
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/wal
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.130:0
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data/info.pb
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.130
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.12.130
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/logs
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:05.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:05.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:05.750108 20995 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:05.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:05.752557 20995 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:05.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:05.807056 20985 heartbeater.cc:503] Master 127.20.12.188:45025 was elected leader, sending a full tablet report...
11:11:05.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.807734 21002 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.826766 21004 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:05.814931 21001 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:07.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:07.548795 21003 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1734 milliseconds
11:11:07.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.549005 20995 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:11:07.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.550864 20995 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:07.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.554075 20995 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:07.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.555579 20995 hybrid_clock.cc:648] HybridClock initialized: now 1786101067555552 us; error 61 us; skew 500 ppm
11:11:07.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.556773 20995 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.565399 20995 webserver.cc:460] Webserver started at http://127.20.12.130:42495/ using document root <none> and password file <none>
11:11:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.566877 20995 fs_manager.cc:359] Metadata directory not provided
11:11:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.567178 20995 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.567830 20995 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:11:07.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.576892 20995 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data/instance:
11:11:07.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cefce422f6d247a896e12b4af1822e2f"
11:11:07.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:07 on dist-test-slave-77v9"
11:11:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.578918 20995 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/wal/instance:
11:11:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cefce422f6d247a896e12b4af1822e2f"
11:11:07.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:07 on dist-test-slave-77v9"
11:11:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.590027 20995 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.006s sys 0.006s
11:11:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.599916 21012 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:07.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.601583 20995 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.002s
11:11:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.602075 20995 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/wal
11:11:07.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cefce422f6d247a896e12b4af1822e2f"
11:11:07.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:07 on dist-test-slave-77v9"
11:11:07.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.602555 20995 fs_report.cc:383] FS layout report
11:11:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/wal
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/wal
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data/data
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:07.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:07.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.661543 20995 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.663710 20995 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.664330 20995 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.668478 20995 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:11:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.676132 20995 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:11:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.676504 20995 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:11:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.676906 20995 ts_tablet_manager.cc:610] Registered 0 tablets
11:11:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:07.677186 20995 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:11:08.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.271198 20995 rpc_server.cc:307] RPC server started. Bound to: 127.20.12.130:44365
11:11:08.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.271791 21126 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.12.130:44365 every 8 connection(s)
11:11:08.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.275173 20995 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/data/info.pb
11:11:08.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.281802 20530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 20995
11:11:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.282521 20530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-1/wal/instance
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.329600 20530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/wal
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/logs
11:11:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data/info.pb
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.131:0
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.12.131
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.131
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:08.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:11:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.394038 21128 heartbeater.cc:347] Connected to a master server at 127.20.12.189:37605
11:11:08.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.394658 21128 heartbeater.cc:464] Registering TS with master...
11:11:08.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.396661 21128 heartbeater.cc:511] Master 127.20.12.189:37605 requested a full tablet report, sending...
11:11:08.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.400382 20652 ts_manager.cc:194] Registered new tserver with Master: cefce422f6d247a896e12b4af1822e2f (127.20.12.130:44365)
11:11:08.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.401357 21127 heartbeater.cc:347] Connected to a master server at 127.20.12.188:45025
11:11:08.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.401803 21127 heartbeater.cc:464] Registering TS with master...
11:11:08.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.403628 21127 heartbeater.cc:511] Master 127.20.12.188:45025 requested a full tablet report, sending...
11:11:08.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.408082 20734 ts_manager.cc:194] Registered new tserver with Master: cefce422f6d247a896e12b4af1822e2f (127.20.12.130:44365)
11:11:08.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.411062 21133 heartbeater.cc:347] Connected to a master server at 127.20.12.190:43375
11:11:08.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.411487 21133 heartbeater.cc:464] Registering TS with master...
11:11:08.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.412058 20734 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.12.130:42115
11:11:08.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.413408 21133 heartbeater.cc:511] Master 127.20.12.190:43375 requested a full tablet report, sending...
11:11:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:08.418480 20576 ts_manager.cc:194] Registered new tserver with Master: cefce422f6d247a896e12b4af1822e2f (127.20.12.130:44365)
11:11:08.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:08.940696 21137 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:08.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:08.941293 21137 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:08.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:08.941900 21137 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:08.975212 21137 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:08.976226 21137 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.12.131
11:11:09.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:09.014537 21137 tablet_server_runner.cc:78] Tablet server non-default flags:
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.12.148:42775
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/wal
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.12.131:0
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data/info.pb
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.12.131
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.12.190:43375,127.20.12.189:37605,127.20.12.188:45025
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.12.131
11:11:09.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/logs
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:09.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:09.016100 21137 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:09.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:09.017874 21137 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:09.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:09.042884 21143 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:09.419420 21127 heartbeater.cc:503] Master 127.20.12.188:45025 was elected leader, sending a full tablet report...
11:11:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:09.045990 21137 server_base.cc:1034] running on GCE node
11:11:10.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:09.045367 21146 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:10.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:09.042923 21144 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.471421 21137 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.475649 21137 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.477105 21137 hybrid_clock.cc:648] HybridClock initialized: now 1786101070477005 us; error 72 us; skew 500 ppm
11:11:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.478263 21137 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:10.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.500198 21137 webserver.cc:460] Webserver started at http://127.20.12.131:34303/ using document root <none> and password file <none>
11:11:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.501547 21137 fs_manager.cc:359] Metadata directory not provided
11:11:10.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.501806 21137 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:10.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.502337 21137 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:11:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.509589 21137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data/instance:
11:11:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "305ad54463e64b2ea34800731816c9de"
11:11:10.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:10 on dist-test-slave-77v9"
11:11:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.511245 21137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/wal/instance:
11:11:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "305ad54463e64b2ea34800731816c9de"
11:11:10.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:10 on dist-test-slave-77v9"
11:11:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.525234 21137 fs_manager.cc:691] Time spent creating directory manager: real 0.013s user 0.005s sys 0.009s
11:11:10.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.538170 21153 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:10.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.539892 21137 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.006s sys 0.002s
11:11:10.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.540385 21137 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/wal
11:11:10.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "305ad54463e64b2ea34800731816c9de"
11:11:10.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:10 on dist-test-slave-77v9"
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.540786 21137 fs_report.cc:383] FS layout report
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/wal
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/wal
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data/data
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:10.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:10.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.607733 21137 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.610973 21137 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.611927 21137 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.621254 21137 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:11:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.630790 21137 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:11:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.631017 21137 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:11:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.631260 21137 ts_tablet_manager.cc:610] Registered 0 tablets
11:11:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:10.631402 21137 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:11:11.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.128007 21137 rpc_server.cc:307] RPC server started. Bound to: 127.20.12.131:46571
11:11:11.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.128360 21269 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.12.131:46571 every 8 connection(s)
11:11:11.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.131572 21137 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/data/info.pb
11:11:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.143927 20530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 21137
11:11:11.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.145975 20530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster7498303477905664664/ts-2/wal/instance
11:11:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.233086 21270 heartbeater.cc:347] Connected to a master server at 127.20.12.188:45025
11:11:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.233771 21270 heartbeater.cc:464] Registering TS with master...
11:11:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.237195 21270 heartbeater.cc:511] Master 127.20.12.188:45025 requested a full tablet report, sending...
11:11:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.239615 21271 heartbeater.cc:347] Connected to a master server at 127.20.12.189:37605
11:11:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.240240 21271 heartbeater.cc:464] Registering TS with master...
11:11:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.241721 20734 ts_manager.cc:194] Registered new tserver with Master: 305ad54463e64b2ea34800731816c9de (127.20.12.131:46571)
11:11:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.241837 21271 heartbeater.cc:511] Master 127.20.12.189:37605 requested a full tablet report, sending...
11:11:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.244707 20734 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.20.12.131:52877
11:11:11.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.246186 20652 ts_manager.cc:194] Registered new tserver with Master: 305ad54463e64b2ea34800731816c9de (127.20.12.131:46571)
11:11:11.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.257027 21274 heartbeater.cc:347] Connected to a master server at 127.20.12.190:43375
11:11:11.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.257683 21274 heartbeater.cc:464] Registering TS with master...
11:11:11.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.258940 21274 heartbeater.cc:511] Master 127.20.12.190:43375 requested a full tablet report, sending...
11:11:11.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.263487 20576 ts_manager.cc:194] Registered new tserver with Master: 305ad54463e64b2ea34800731816c9de (127.20.12.131:46571)
11:11:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:11.269747 20530 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
11:11:11.300 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:11:11.301 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
11:11:11.315 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.12.190"
port: 43375
}
bound_http_address {
host: "127.20.12.190"
port: 39361
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.12.189"
port: 37605
}
bound_http_address {
host: "127.20.12.189"
port: 35113
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.12.188"
port: 45025
}
bound_http_address {
host: "127.20.12.188"
port: 44359
}
}
}
11:11:11.323 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:11:11.325 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.20.12.129"
port: 35605
}
bound_http_address {
host: "127.20.12.129"
port: 42675
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.20.12.130"
port: 44365
}
bound_http_address {
host: "127.20.12.130"
port: 42495
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.20.12.131"
port: 46571
}
bound_http_address {
host: "127.20.12.131"
port: 34303
}
}
}
11:11:11.326 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
11:11:11.583 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
11:11:11.765 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskV256yr/test-tmp
11:11:11.766 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:11:11.767 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskV256yr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:11:11.771 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
11:11:12.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:12.189 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:12.193357 21282 mini_oidc.cc:152] Starting JWKS server
11:11:12.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:12.199713 21282 webserver.cc:460] Webserver started at http://127.0.0.1:41569/ using document root <none> and password file <none>
11:11:12.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:12.200352 21282 mini_oidc.cc:189] Starting OIDC Discovery server
11:11:12.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:12.204224 21282 webserver.cc:460] Webserver started at http://127.0.0.1:38707/ using document root <none> and password file <none>
11:11:12.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:12.255137 21270 heartbeater.cc:503] Master 127.20.12.188:45025 was elected leader, sending a full tablet report...
11:11:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
11:11:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/principal' for realm 'KRBTEST.COM',
11:11:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
11:11:12.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:12 dist-test-slave-77v9 krb5kdc[21294](info): setting up network...
11:11:12.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
11:11:12.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:12 dist-test-slave-77v9 krb5kdc[21294](info): set up 2 sockets
11:11:12.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:12 dist-test-slave-77v9 krb5kdc[21294](info): commencing operation
11:11:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
11:11:13.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:13.470443 21282 mini_kdc.cc:121] Time spent starting KDC: real 1.232s user 0.002s sys 0.047s
11:11:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
11:11:13.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:11:13.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
11:11:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
11:11:13.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:11:13.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
11:11:13.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
11:11:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:11:13.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
11:11:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:11:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/test-user.keytab.
11:11:13.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/test-user.keytab.
11:11:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:13 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101073, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:13.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:11:13.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T11:11:13Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:11:13.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T11:11:13Z Disabled control of system clock
11:11:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.200.190@KRBTEST.COM; defaulting to no policy
11:11:13.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:11:13.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.200.190@KRBTEST.COM" created.
11:11:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:11:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.200.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.190.keytab.
11:11:13.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.200.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.190.keytab.
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:13.754798 21282 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/logs
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/info.pb
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:13.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.190:44227
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.190
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.190.keytab
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.190
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:13.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:11:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.236800 21308 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:14.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.237499 21308 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:14.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.238265 21308 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.278380 21308 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:11:14.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.278739 21308 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:11:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.279143 21308 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.279381 21308 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:11:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.279588 21308 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:11:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.279788 21308 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:11:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.279937 21308 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41569/jwks.json
11:11:14.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:14.322449 21308 master_runner.cc:386] Master server non-default flags:
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.190.keytab
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.190
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.190:44227
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/info.pb
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.190
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/logs
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:14.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:14.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:14.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:14.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:14.323733 21308 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:14.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:14.324802 21308 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:14.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.345492 21320 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.345854 21317 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:14.346092 21318 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:14.348549 21308 server_base.cc:1034] running on GCE node
11:11:15.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.660086 21308 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:15.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.663672 21308 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:15.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.665126 21308 hybrid_clock.cc:648] HybridClock initialized: now 1786101075665090 us; error 44 us; skew 500 ppm
11:11:15.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.666198 21308 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:15 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101075, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.680894 21308 init.cc:377] Logged in from keytab as kudu/127.20.200.190@KRBTEST.COM (short username kudu)
11:11:15.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.688465 21308 webserver.cc:460] Webserver started at http://127.20.200.190:44091/ using document root <none> and password file <none>
11:11:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.689489 21308 fs_manager.cc:359] Metadata directory not provided
11:11:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.689685 21308 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.690212 21308 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:11:15.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.695858 21308 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/instance:
11:11:15.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0de10f343f8a4b679589f5d1f589ca4d"
11:11:15.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:15 on dist-test-slave-77v9"
11:11:15.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.697083 21308 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal/instance:
11:11:15.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0de10f343f8a4b679589f5d1f589ca4d"
11:11:15.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:15 on dist-test-slave-77v9"
11:11:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.707268 21308 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.003s sys 0.005s
11:11:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.716084 21329 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.717568 21308 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.006s sys 0.001s
11:11:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.718007 21308 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0de10f343f8a4b679589f5d1f589ca4d"
11:11:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:15 on dist-test-slave-77v9"
11:11:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.718405 21308 fs_report.cc:383] FS layout report
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/data
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:15.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.776198 21308 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:15.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.784516 21308 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:15.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:15.785172 21308 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.080564 21308 rpc_server.cc:307] RPC server started. Bound to: 127.20.200.190:44227
11:11:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.081115 21381 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.200.190:44227 every 8 connection(s)
11:11:16.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.083904 21308 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/info.pb
11:11:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.093237 21282 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 21308
11:11:16.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.094240 21282 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal/instance
11:11:16.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.098887 21382 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:11:16.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.124079 21382 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } has no permanent_uuid. Determining permanent_uuid...
11:11:16.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.200.189@KRBTEST.COM; defaulting to no policy
11:11:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:11:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.200.189@KRBTEST.COM" created.
11:11:16.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:16 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101075, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.190@KRBTEST.COM for kudu/127.20.200.190@KRBTEST.COM
11:11:16.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.264406 21382 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } has no permanent_uuid. Determining permanent_uuid...
11:11:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.270819 21332 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.277555 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:11:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.200.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.189.keytab.
11:11:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.200.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.189.keytab.
11:11:16.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.292886 21282 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.20.200.189: real 0.192s user 0.003s sys 0.039s
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.294987 21282 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/logs
11:11:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/info.pb
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.189:42931
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.189
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.189.keytab
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.189
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:11:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:16.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:11:16.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.327255 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 1
11:11:16.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.330963 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:16.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.399358 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 2
11:11:16.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.407094 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:16.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.498656 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 3
11:11:16.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.504105 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:16.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.647945 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 4
11:11:16.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.655050 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:16.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.910450 21389 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:16.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.911253 21389 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:16.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.912174 21389 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.949031 21389 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:11:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.949292 21389 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:11:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.949590 21389 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.949807 21389 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:11:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.950017 21389 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:11:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.950224 21389 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:11:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.950387 21389 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41569/jwks.json
11:11:16.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.954810 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 5
11:11:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:16.961633 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:16.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.996484 21389 master_runner.cc:386] Master server non-default flags:
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.189.keytab
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.189
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.189:42931
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/info.pb
11:11:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.189
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/logs
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:16.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.997787 21389 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:16.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:16.999014 21389 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:17.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:17.035748 21396 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:17.509200 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 6
11:11:17.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:17.535617 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:18.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.596614 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 7
11:11:18.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:18.645951 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.189:42931: Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:17.037350 21395 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:18.820255 21398 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:18.822218 21397 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1785 milliseconds
11:11:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.822348 21389 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:11:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.823993 21389 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.827585 21389 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:18.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.828981 21389 hybrid_clock.cc:648] HybridClock initialized: now 1786101078828958 us; error 64 us; skew 500 ppm
11:11:18.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.830027 21389 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:18.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:18 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101078, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:18.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.842512 21389 init.cc:377] Logged in from keytab as kudu/127.20.200.189@KRBTEST.COM (short username kudu)
11:11:18.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.857776 21389 webserver.cc:460] Webserver started at http://127.20.200.189:45121/ using document root <none> and password file <none>
11:11:18.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.858944 21389 fs_manager.cc:359] Metadata directory not provided
11:11:18.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.859153 21389 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:18.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.859549 21389 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:11:18.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.866061 21389 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/instance:
11:11:18.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "282c905f1f3c41cbb7fbf11612889868"
11:11:18.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:18 on dist-test-slave-77v9"
11:11:18.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.867410 21389 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal/instance:
11:11:18.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "282c905f1f3c41cbb7fbf11612889868"
11:11:18.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:18 on dist-test-slave-77v9"
11:11:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.881809 21389 fs_manager.cc:691] Time spent creating directory manager: real 0.013s user 0.014s sys 0.001s
11:11:18.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.895213 21411 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.896641 21389 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.001s sys 0.008s
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.897133 21389 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "282c905f1f3c41cbb7fbf11612889868"
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:18 on dist-test-slave-77v9"
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.897626 21389 fs_report.cc:383] FS layout report
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/data
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:18.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:18.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:18.922530 21266 debug-util.cc:398] Leaking SignalData structure 0x7b08000bba60 after lost signal to thread 21138
11:11:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:18.924793 21266 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae3c0 after lost signal to thread 21269
11:11:18.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.950717 21389 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.957675 21389 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:18.958295 21389 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:19.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.186241 21389 rpc_server.cc:307] RPC server started. Bound to: 127.20.200.189:42931
11:11:19.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.186605 21465 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.200.189:42931 every 8 connection(s)
11:11:19.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.192096 21389 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/info.pb
11:11:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.195443 21282 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 21389
11:11:19.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.195938 21282 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal/instance
11:11:19.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.215965 21467 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:11:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.200.188@KRBTEST.COM; defaulting to no policy
11:11:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:11:19.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.200.188@KRBTEST.COM" created.
11:11:19.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.251474 21467 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } has no permanent_uuid. Determining permanent_uuid...
11:11:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:11:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.200.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.188.keytab.
11:11:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.200.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.188.keytab.
11:11:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.312458 21282 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.20.200.188: real 0.114s user 0.008s sys 0.028s
11:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.316103 21282 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data
11:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/logs
11:11:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/info.pb
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.188:45753
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.188
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.188.keytab
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.188
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:19.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:11:19.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:19 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101078, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.189@KRBTEST.COM for kudu/127.20.200.190@KRBTEST.COM
11:11:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.364212 21467 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } has no permanent_uuid. Determining permanent_uuid...
11:11:19.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:19 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101078, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.189@KRBTEST.COM for kudu/127.20.200.189@KRBTEST.COM
11:11:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.455212 21467 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } has no permanent_uuid. Determining permanent_uuid...
11:11:19.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.460465 21414 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:19.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.477835 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:19.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.527472 21467 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 1
11:11:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.534260 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.602921 21467 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 2
11:11:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.612182 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:19.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.703883 21467 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 3
11:11:19.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.715306 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:19.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:19.858994 21467 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 4
11:11:19.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.865473 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:19.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.940305 21474 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:19.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.941335 21474 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:19.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:19.942574 21474 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.013439 21474 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:11:20.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.013803 21474 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:11:20.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.014299 21474 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:20.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.014667 21474 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:11:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.015019 21474 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:11:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.015372 21474 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:11:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.015661 21474 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41569/jwks.json
11:11:20.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.068228 21474 master_runner.cc:386] Master server non-default flags:
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.188.keytab
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.188
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.188:45753
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:11:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/info.pb
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.188
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/logs
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.069788 21474 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.071103 21474 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:20.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.121673 21483 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.165036 21467 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 5
11:11:20.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.173992 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:20.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.722466 21467 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 6
11:11:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.736614 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } attempt: 8
11:11:20.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.758036 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:20 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101075, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.190@KRBTEST.COM for kudu/127.20.200.189@KRBTEST.COM
11:11:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.814823 21382 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } has no permanent_uuid. Determining permanent_uuid...
11:11:20.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.831841 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.897482 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 1
11:11:20.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.905714 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:20.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.959224 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 2
11:11:20.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.964906 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:21.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.041529 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 3
11:11:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:21.048918 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.204607 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 4
11:11:21.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:21.215297 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:21.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.512041 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 5
11:11:21.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:21.516731 21331 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111) [suppressed 12 similar messages]
11:11:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:21.521946 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:21.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.136261 21482 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:20.140548 21474 server_base.cc:1034] running on GCE node
11:11:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:20.141268 21485 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:21.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.796023 21474 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.800570 21474 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.802050 21474 hybrid_clock.cc:648] HybridClock initialized: now 1786101081802006 us; error 75 us; skew 500 ppm
11:11:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.802899 21474 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:21.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:21 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101081, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:21.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.809593 21474 init.cc:377] Logged in from keytab as kudu/127.20.200.188@KRBTEST.COM (short username kudu)
11:11:21.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.815514 21474 webserver.cc:460] Webserver started at http://127.20.200.188:39931/ using document root <none> and password file <none>
11:11:21.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.816522 21474 fs_manager.cc:359] Metadata directory not provided
11:11:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.816727 21474 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:21.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.817421 21474 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:11:21.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.818552 21467 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 7
11:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.825958 21474 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/instance:
11:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f767ee892a6440b693dee13c467709ad"
11:11:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:21 on dist-test-slave-77v9"
11:11:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.827693 21474 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal/instance:
11:11:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f767ee892a6440b693dee13c467709ad"
11:11:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:21 on dist-test-slave-77v9"
11:11:21.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:21.833487 21467 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:21.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.839267 21474 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.009s sys 0.004s
11:11:21.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.847791 21506 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:21.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.849339 21474 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.002s sys 0.004s
11:11:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.849805 21474 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f767ee892a6440b693dee13c467709ad"
11:11:21.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:21 on dist-test-slave-77v9"
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.850219 21474 fs_report.cc:383] FS layout report
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/data
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:21.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.914323 21474 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.922569 21474 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:21.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:21.923367 21474 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.043597 21382 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } attempt: 6
11:11:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:22.059532 21382 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.20.200.188:45753: Network error: Client connection negotiation failed: client connection to 127.20.200.188:45753: connect: Connection refused (error 111)
11:11:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.168919 21474 rpc_server.cc:307] RPC server started. Bound to: 127.20.200.188:45753
11:11:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.169888 21560 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.200.188:45753 every 8 connection(s)
11:11:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.171797 21474 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/info.pb
11:11:22.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.173442 21282 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 21474
11:11:22.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.174276 21282 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal/instance
11:11:22.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.191650 21561 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:11:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.236769 21561 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } has no permanent_uuid. Determining permanent_uuid...
11:11:22.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101073, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.200.188@KRBTEST.COM
11:11:22.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101081, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for kudu/127.20.200.190@KRBTEST.COM
11:11:22.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.324035 21561 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } has no permanent_uuid. Determining permanent_uuid...
11:11:22.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101073, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.200.189@KRBTEST.COM
11:11:22.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101081, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for kudu/127.20.200.189@KRBTEST.COM
11:11:22.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.375277 21561 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } has no permanent_uuid. Determining permanent_uuid...
11:11:22.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101073, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.200.190@KRBTEST.COM
11:11:22.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101081, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for kudu/127.20.200.188@KRBTEST.COM
11:11:22.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.415948 21282 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
11:11:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.428905 21561 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Bootstrap starting.
11:11:22.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:22.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.442807 21561 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Neither blocks nor log segments found. Creating new log.
11:11:22.444 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
11:11:22.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.448374 21561 log.cc:824] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Log is configured to *not* fsync() on all Append() calls
11:11:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.458508 21561 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: No bootstrap required, opened a new log
11:11:22.460 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
11:11:22.461 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
11:11:22.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.200.190"
port: 44227
}
bound_http_address {
host: "127.20.200.190"
port: 44091
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.200.189"
port: 42931
}
bound_http_address {
host: "127.20.200.189"
port: 45121
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.200.188"
port: 45753
}
bound_http_address {
host: "127.20.200.188"
port: 39931
}
}
}
11:11:22.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:11:22.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:22.465 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:11:22.467 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:11:22.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.510092 21561 raft_consensus.cc:348] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:22.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.510882 21561 raft_consensus.cc:374] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:11:22.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101082, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:22.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.511170 21561 raft_consensus.cc:729] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f767ee892a6440b693dee13c467709ad, State: Initialized, Role: FOLLOWER
11:11:22.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:11:22.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.512286 21561 consensus_queue.cc:260] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:22.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.519410 21573 sys_catalog.cc:455] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } } }
11:11:22.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.521981 21573 sys_catalog.cc:458] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [sys.catalog]: This master's current role is: FOLLOWER
11:11:22.522 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:22.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.522517 21561 sys_catalog.cc:564] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [sys.catalog]: configured and running, proceeding with master startup.
11:11:22.531 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc
11:11:22.544 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:11:22.546 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc
11:11:22.549 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:11:22.558 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
11:11:22.565 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:11:22.566 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:11:22.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.608654 21585 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:11:22.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:22 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101082, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:22.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:11:22.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:22.637 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc
11:11:22.639 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:11:22.640 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.200.189:42931
11:11:22.643 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:11:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.643864 21282 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 21389
11:11:22.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.614738 21474 master_runner.cc:186] Error getting master registration for 127.20.200.190:44227: OK, instance_id { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" instance_seqno: 1786101075996467 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:11:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.654995 21474 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:11:22.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:22.670 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.200.190:44227
11:11:22.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:11:22.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.671074 21282 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 21308
11:11:22.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:22.689976 21512 tablet.cc:2369] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:11:22.704 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:22.704 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.200.188:45753
11:11:22.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:11:22.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.705655 21282 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 21474
11:11:22.733 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:22.733 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.200.189:42931
11:11:22.735 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:22.736771 21282 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/logs
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/info.pb
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.189:42931
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.189
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45121
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.189.keytab
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.189
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:22.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:11:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.064491 21587 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.065237 21587 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.065987 21587 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.100348 21587 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:11:23.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.100653 21587 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:11:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.101105 21587 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.101454 21587 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:11:23.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.101775 21587 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:11:23.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.102103 21587 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:11:23.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.102339 21587 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41569/jwks.json
11:11:23.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:23.148474 21587 master_runner.cc:386] Master server non-default flags:
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.189.keytab
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.189
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.189:42931
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:23.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/info.pb
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.189
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45121
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/logs
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:23.150044 21587 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:23.151252 21587 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:23.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.169791 21596 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:24.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.169862 21594 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:24.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:23.169947 21593 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:24.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:23.171464 21587 server_base.cc:1034] running on GCE node
11:11:24.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.470434 21587 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:24.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.473548 21587 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:24.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.475068 21587 hybrid_clock.cc:648] HybridClock initialized: now 1786101084475007 us; error 95 us; skew 500 ppm
11:11:24.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.476099 21587 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:24.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:24 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101084, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:24.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.483425 21587 init.cc:377] Logged in from keytab as kudu/127.20.200.189@KRBTEST.COM (short username kudu)
11:11:24.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.489678 21587 webserver.cc:460] Webserver started at http://127.20.200.189:45121/ using document root <none> and password file <none>
11:11:24.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.490731 21587 fs_manager.cc:359] Metadata directory not provided
11:11:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.490950 21587 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:24.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.499754 21587 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
11:11:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.504961 21605 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.506507 21587 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.507114 21587 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "282c905f1f3c41cbb7fbf11612889868"
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:18 on dist-test-slave-77v9"
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.510177 21587 fs_report.cc:383] FS layout report
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal
11:11:24.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/data
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:24.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:24.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.547729 21587 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:24.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.554280 21587 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:24.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.554723 21587 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:24.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.633425 21587 rpc_server.cc:307] RPC server started. Bound to: 127.20.200.189:42931
11:11:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.633543 21656 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.200.189:42931 every 8 connection(s)
11:11:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.636552 21587 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/data/info.pb
11:11:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.640977 21282 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 21587
11:11:24.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:24.641 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.200.190:44227
11:11:24.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:11:24.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.643231 21282 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:24.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:24.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:24.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data
11:11:24.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:24.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:24.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:24.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:24.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:24.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:24.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/logs
11:11:24.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/info.pb
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.190:44227
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.190
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.190.keytab
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.190
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:11:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:24.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:11:24.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.650774 21657 sys_catalog.cc:263] Verifying existing consensus state
11:11:24.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260807 11:11:24.652052 21657 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:11:24.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.652594 21587 master.cc:537] Master@127.20.200.189:42931 shutting down...
11:11:24.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:24.733999 21587 master.cc:552] Master@127.20.200.189:42931 shutdown complete.
11:11:24.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:11:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.008630 21659 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:25.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.009486 21659 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:25.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.010411 21659 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.046725 21659 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:11:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.047123 21659 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:11:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.047513 21659 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.047750 21659 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:11:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.047962 21659 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:11:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.048169 21659 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:11:25.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.048346 21659 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41569/jwks.json
11:11:25.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:25.088385 21659 master_runner.cc:386] Master server non-default flags:
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.190.keytab
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.190
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.190:44227
11:11:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/info.pb
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.190
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44091
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/logs
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:25.089561 21659 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:25.090755 21659 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:25.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.108891 21668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:26.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.108947 21666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:25.109820 21665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:26.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:25.111387 21659 server_base.cc:1034] running on GCE node
11:11:26.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.411548 21659 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:26.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.414453 21659 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:26.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.415889 21659 hybrid_clock.cc:648] HybridClock initialized: now 1786101086415848 us; error 47 us; skew 500 ppm
11:11:26.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.416869 21659 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:26.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:26 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101086, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.426949 21659 init.cc:377] Logged in from keytab as kudu/127.20.200.190@KRBTEST.COM (short username kudu)
11:11:26.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.433519 21659 webserver.cc:460] Webserver started at http://127.20.200.190:44091/ using document root <none> and password file <none>
11:11:26.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.434557 21659 fs_manager.cc:359] Metadata directory not provided
11:11:26.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.434772 21659 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:26.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.444105 21659 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.002s
11:11:26.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.449525 21677 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:26.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.450923 21659 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
11:11:26.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.451249 21659 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0de10f343f8a4b679589f5d1f589ca4d"
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:15 on dist-test-slave-77v9"
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.453756 21659 fs_report.cc:383] FS layout report
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/data
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:26.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.509092 21659 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:26.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.517180 21659 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:26.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.517657 21659 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.603910 21659 rpc_server.cc:307] RPC server started. Bound to: 127.20.200.190:44227
11:11:26.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.603997 21729 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.200.190:44227 every 8 connection(s)
11:11:26.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.608439 21659 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/data/info.pb
11:11:26.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.613716 21282 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 21659
11:11:26.615 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:26.615 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.200.188:45753
11:11:26.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.617913 21282 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskV256yr/build/tsan/bin/kudu
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/logs
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/info.pb
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:11:26.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.188:45753
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.188
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39931
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.188.keytab
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.188
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:26.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:26.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:11:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.626556 21730 sys_catalog.cc:263] Verifying existing consensus state
11:11:26.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260807 11:11:26.627354 21730 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:11:26.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.627774 21659 master.cc:537] Master@127.20.200.190:44227 shutting down...
11:11:26.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:26.700102 21659 master.cc:552] Master@127.20.200.190:44227 shutdown complete.
11:11:26.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:11:26.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:26.975142 21732 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:11:26.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:26.975713 21732 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:11:26.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:26.976325 21732 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:11:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.009788 21732 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:11:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.010039 21732 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:11:27.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.010448 21732 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:11:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.010763 21732 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:11:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.011004 21732 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:11:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.011224 21732 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:11:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.011597 21732 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41569/jwks.json
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:27.049932 21732 master_runner.cc:386] Master server non-default flags:
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.200.148:45569
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.200.190:44227,127.20.200.189:42931,127.20.200.188:45753
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/krb5kdc/kudu_127.20.200.188.keytab
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.200.188
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.200.188:45753
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41569/jwks.json
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:11:27.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/info.pb
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.200.188
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39931
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/logs
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 07 Aug 2026 10:26:28 UTC on ccycloud.quasar-pbxjyx.root.comops.site
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4572144
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:11:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:27.051116 21732 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:11:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:27.052101 21732 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:11:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.070850 21738 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.070919 21739 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:27.073529 21732 server_base.cc:1034] running on GCE node
11:11:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:27.073477 21741 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:11:28.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.333384 21732 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:11:28.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.336366 21732 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:11:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.337778 21732 hybrid_clock.cc:648] HybridClock initialized: now 1786101088337758 us; error 50 us; skew 500 ppm
11:11:28.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.338635 21732 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:11:28.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:28 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101088, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.348412 21732 init.cc:377] Logged in from keytab as kudu/127.20.200.188@KRBTEST.COM (short username kudu)
11:11:28.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.354282 21732 webserver.cc:460] Webserver started at http://127.20.200.188:39931/ using document root <none> and password file <none>
11:11:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.356089 21732 fs_manager.cc:359] Metadata directory not provided
11:11:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.356447 21732 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:11:28.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.367969 21732 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.001s sys 0.004s
11:11:28.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.373394 21751 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:11:28.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.374958 21732 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
11:11:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.375384 21732 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data,/tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f767ee892a6440b693dee13c467709ad"
11:11:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-07 11:11:21 on dist-test-slave-77v9"
11:11:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.379112 21732 fs_report.cc:383] FS layout report
11:11:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:11:28.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/wal
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/data
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:11:28.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:11:28.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.423113 21732 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:11:28.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.429591 21732 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:11:28.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.430025 21732 kserver.cc:163] Server-wide thread pool size limit: 3276
11:11:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.526611 21732 rpc_server.cc:307] RPC server started. Bound to: 127.20.200.188:45753
11:11:28.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.526845 21803 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.20.200.188:45753 every 8 connection(s)
11:11:28.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.529973 21732 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskV256yr/test-tmp/mini-kudu-cluster437275675555544666/master-2/data/info.pb
11:11:28.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.539637 21282 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskV256yr/build/tsan/bin/kudu as pid 21732
11:11:28.540 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:11:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.542174 21804 sys_catalog.cc:263] Verifying existing consensus state
11:11:28.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.548382 21804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Bootstrap starting.
11:11:28.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.592424 21804 log.cc:824] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Log is configured to *not* fsync() on all Append() calls
11:11:28.595 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:28.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.599077 21804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:11:28.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.600385 21804 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Bootstrap complete.
11:11:28.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.632010 21804 raft_consensus.cc:348] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:28.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.635741 21804 raft_consensus.cc:374] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:11:28.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.636147 21804 raft_consensus.cc:729] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f767ee892a6440b693dee13c467709ad, State: Initialized, Role: FOLLOWER
11:11:28.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.637310 21804 consensus_queue.cc:260] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:28.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.640425 21808 sys_catalog.cc:455] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } } }
11:11:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.641867 21808 sys_catalog.cc:458] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [sys.catalog]: This master's current role is: FOLLOWER
11:11:28.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.642534 21804 sys_catalog.cc:564] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [sys.catalog]: configured and running, proceeding with master startup.
11:11:28.675 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:28.676 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.200.188:45753(127.20.200.188:45753)] connecting to peer
11:11:28.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:28.685817 21757 tablet.cc:2369] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:11:28.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:28.723557 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:28.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:28.725313 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.682741 21732 master_runner.cc:186] Error getting master registration for 127.20.200.190:44227: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111),
11:11:28.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.730321 21732 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:11:28.740 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:28.741 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:28.743 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:28.746 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:28.747 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:28.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:28.750 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.200.188:45753(127.20.200.188:45753)] Successfully connected to peer
11:11:28.843 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
11:11:28.852 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.200.188
11:11:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.953850 21808 raft_consensus.cc:482] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:11:28.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.954495 21808 raft_consensus.cc:504] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.960602 21808 leader_election.cc:288] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227), 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931)
11:11:28.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:28.964250 21755 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111)
11:11:28.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:28.988731 21754 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931): Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:28.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:28.998171 21755 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227): Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111)
11:11:28.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.998581 21755 leader_election.cc:302] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f767ee892a6440b693dee13c467709ad; no voters: 0de10f343f8a4b679589f5d1f589ca4d, 282c905f1f3c41cbb7fbf11612889868
11:11:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:28.999456 21808 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:11:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:29 dist-test-slave-77v9 krb5kdc[21294](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101082, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.200.188@KRBTEST.COM
11:11:29.163 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xa541f7cf, L:/127.0.0.1:55338 - R:/127.20.200.188:45753] using SASL/GSSAPI
11:11:29.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.207 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.209 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.214 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.215 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.215 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.220 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.220 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.221 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.225 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.227 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.237 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.247 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.250 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.250 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.253 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.264 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.264 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.272 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.273 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.287 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.294 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.326 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.336 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.336 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.337 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.337 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.367 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.374 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.378 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.505 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.507 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.513 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.513 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.514 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.516 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:29.729179 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:29.729445 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:29.786 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:29.788 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:29.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:29.791 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.791 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.791 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:29.794 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:30.166 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:30.168 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:30.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:30.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:30.170 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:30.171 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:30.173 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:30.174 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:30.175 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:30.732545 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:30.732834 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:30.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:30.747 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:30.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:30.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:30.748 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:30.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:30.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:30.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:30.752 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:30.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:30.843132 21828 raft_consensus.cc:482] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:11:30.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:30.843664 21828 raft_consensus.cc:504] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:30.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:30.845573 21828 leader_election.cc:288] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227), 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931)
11:11:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:30.852397 21755 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227): Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111)
11:11:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:30.853008 21754 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931): Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:30.853386 21754 leader_election.cc:302] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f767ee892a6440b693dee13c467709ad; no voters: 0de10f343f8a4b679589f5d1f589ca4d, 282c905f1f3c41cbb7fbf11612889868
11:11:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:30.854045 21828 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:11:31.185 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:31.187 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:31.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:31.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:31.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:31.190 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:31.190 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:31.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:31.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:31.735927 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:31.736222 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:32.085 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:32.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:32.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:32.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:32.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:32.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:32.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:32.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:32.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:32.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:32.665246 21839 raft_consensus.cc:482] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:11:32.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:32.665696 21839 raft_consensus.cc:504] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:32.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:32.667434 21839 leader_election.cc:288] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227), 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931)
11:11:32.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:32.672850 21755 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227): Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111)
11:11:32.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:32.673694 21754 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931): Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:32.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:32.674062 21754 leader_election.cc:302] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f767ee892a6440b693dee13c467709ad; no voters: 0de10f343f8a4b679589f5d1f589ca4d, 282c905f1f3c41cbb7fbf11612889868
11:11:32.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:32.674721 21839 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:11:32.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:32.739212 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:32.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:32.739475 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:33.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:33.733394 21732 master_runner.cc:186] Error getting master registration for 127.20.200.190:44227: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111),
11:11:33.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:33.733927 21732 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:11:33.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:33.742422 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:33.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:33.742671 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:34.746122 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:34.746467 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:35.126 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:35.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:35.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:35.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:35.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:35.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:35.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:35.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:35.133 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:35.749754 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:35.750044 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:36.753338 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:36.753674 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:37.273741 21848 raft_consensus.cc:482] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:11:37.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:37.274430 21848 raft_consensus.cc:504] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:37.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:37.276584 21848 leader_election.cc:288] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227), 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931)
11:11:37.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:37.279774 21755 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111) [suppressed 5 similar messages]
11:11:37.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:37.282723 21754 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931): Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:37.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:37.283281 21755 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227): Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111)
11:11:37.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:37.283588 21755 leader_election.cc:302] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f767ee892a6440b693dee13c467709ad; no voters: 0de10f343f8a4b679589f5d1f589ca4d, 282c905f1f3c41cbb7fbf11612889868
11:11:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:37.284188 21848 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:11:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:37.757503 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:37.757882 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:38.206 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:38.207 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:38.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:38.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:38.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:38.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:38.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:38.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:38.213 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:38 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101098, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:38.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:38.352718 21747 init.cc:295] Successfully reacquired a new kerberos TGT
11:11:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:38.737757 21732 master_runner.cc:186] Error getting master registration for 127.20.200.190:44227: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111),
11:11:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:38.738207 21732 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:11:38.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:38.761437 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:38.761749 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:39.666 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:39.667 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:39.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:39.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:39.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:39.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:39.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:39.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:39.673 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:39.765892 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:39.766227 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:40.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:40.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:40.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:40.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:40.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:40.050 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:40.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:40.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:40.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:40.769927 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:40.770212 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:41.773099 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:41.773381 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:41.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:41.774166 21856 raft_consensus.cc:482] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:11:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:41.774617 21856 raft_consensus.cc:504] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:41.776367 21856 leader_election.cc:288] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227), 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931)
11:11:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:41.781850 21755 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227): Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111)
11:11:41.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:41.782400 21754 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931): Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:41.782840 21754 leader_election.cc:302] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f767ee892a6440b693dee13c467709ad; no voters: 0de10f343f8a4b679589f5d1f589ca4d, 282c905f1f3c41cbb7fbf11612889868
11:11:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:41.783604 21856 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:11:42.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:42.776667 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:42.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:42.777119 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:43.740921 21732 master_runner.cc:186] Error getting master registration for 127.20.200.190:44227: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111),
11:11:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:43.741276 21732 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:11:43.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:43.780099 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:43.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:43.780359 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:44.006 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:44.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:44.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:44.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:44.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:44.010 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:44.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:44.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:44.015 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:44.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:44.109350 20685 debug-util.cc:398] Leaking SignalData structure 0x7b08000839a0 after lost signal to thread 20618
11:11:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:44.110188 20685 debug-util.cc:398] Leaking SignalData structure 0x7b080003e700 after lost signal to thread 20688
11:11:44.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:44.783285 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:44.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:44.783542 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:45.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:45.786517 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:45.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:45.786777 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:46.025 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:46.027 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:46.028 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:46.028 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:46.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:46.031 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:46.031 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:46.031 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:46.034 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:46.789762 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:46.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:46.790026 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:47.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:47.794358 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:47.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:47.794727 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:48.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:48.248 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:48.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:48.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:48.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:48.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:48.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:48.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:48.255 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:48.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:48 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101108, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.355391 21747 init.cc:295] Successfully reacquired a new kerberos TGT
11:11:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.744755 21732 master_runner.cc:186] Error getting master registration for 127.20.200.190:44227: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111),
11:11:48.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.745255 21732 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:11:48.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.779343 21873 raft_consensus.cc:482] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:11:48.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.779882 21873 raft_consensus.cc:504] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0de10f343f8a4b679589f5d1f589ca4d" member_type: VOTER last_known_addr { host: "127.20.200.190" port: 44227 } } peers { permanent_uuid: "282c905f1f3c41cbb7fbf11612889868" member_type: VOTER last_known_addr { host: "127.20.200.189" port: 42931 } } peers { permanent_uuid: "f767ee892a6440b693dee13c467709ad" member_type: VOTER last_known_addr { host: "127.20.200.188" port: 45753 } }
11:11:48.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.782527 21873 leader_election.cc:288] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227), 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931)
11:11:48.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:48.783916 21755 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111) [suppressed 3 similar messages]
11:11:48.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:48.787858 21754 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 282c905f1f3c41cbb7fbf11612889868 (127.20.200.189:42931): Network error: Client connection negotiation failed: client connection to 127.20.200.189:42931: connect: Connection refused (error 111)
11:11:48.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:48.788308 21755 leader_election.cc:334] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0de10f343f8a4b679589f5d1f589ca4d (127.20.200.190:44227): Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111)
11:11:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.788630 21755 leader_election.cc:302] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f767ee892a6440b693dee13c467709ad; no voters: 0de10f343f8a4b679589f5d1f589ca4d, 282c905f1f3c41cbb7fbf11612889868
11:11:48.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:48.789264 21873 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:11:48.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:48.797859 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:48.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:48.798138 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:49.801088 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:49.801378 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:50.726 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:50.727 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:50.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:50.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:50.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:50.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:50.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:50.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:50.733 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:50.804268 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:50.804543 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:51.807704 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:51.808005 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:52.811595 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:52.811929 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:53.748196 21732 master_runner.cc:186] Error getting master registration for 127.20.200.190:44227: Network error: Client connection negotiation failed: client connection to 127.20.200.190:44227: connect: Connection refused (error 111),
11:11:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:53.748636 21732 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:11:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:53.815259 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:53.815564 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:54.564596 21266 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbf60 after lost signal to thread 21138
11:11:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:54.565620 21266 debug-util.cc:398] Leaking SignalData structure 0x7b08000aea00 after lost signal to thread 21269
11:11:54.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:54.647 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:54.647 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:54.647 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:54.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:54.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:54.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:54.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:54.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:54.765 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:54.766 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:54.767 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:54.767 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:54.767 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:54.768 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:54.768 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:54.768 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:54.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:54.818686 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:54.818971 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:55.822139 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:55.822407 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:56.825244 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:56.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:56.825507 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:57.829217 21820 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f767ee892a6440b693dee13c467709ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:11:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260807 11:11:57.829563 21820 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:11:58.286 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:58.287 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:58.287 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:58.287 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:58.287 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:58.288 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:58.288 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:58.288 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:58.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:58.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 07 11:11:58 dist-test-slave-77v9 krb5kdc[21294](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786101118, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:11:58.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:58.358314 21747 init.cc:295] Successfully reacquired a new kerberos TGT
11:11:58.446 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.189:42931(127.20.200.189:42931)] connecting to peer
11:11:58.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.20.200.190:44227(127.20.200.190:44227)] connecting to peer
11:11:58.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:58.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.200.189:42931(127.20.200.189:42931)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:58.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.189:42931: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931
11:11:58.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:58.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.190:44227(127.20.200.190:44227)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:58.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.20.200.190:44227: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
11:11:58.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753; will retry
11:11:58.458 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29908), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [634ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [639ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [667ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [671ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [673ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [680ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [681ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [688ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [702ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [710ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [716ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [725ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [740ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [745ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [781ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [787ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [789ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [821ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [827ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [829ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [960ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [966ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [967ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1237ms] refreshing cache from master, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1246ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1617ms] refreshing cache from master, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1625ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1627ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1627ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2202ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2204ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2637ms] refreshing cache from master, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2644ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2645ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [3537ms] refreshing cache from master, [3537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [3539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [3540ms] too many traces: truncated at 100 traces, deferred=Deferred@403535664(state=PENDING, result=null, callback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060), errback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060)))
11:11:58.464 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.200.188:45753(127.20.200.188:45753)] disconnecting while in state READY
11:11:58.465 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.200.190:44227(127.20.200.190:44227)] disconnecting while in state TERMINATED
11:11:58.465 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.200.189:42931(127.20.200.189:42931)] disconnecting while in state TERMINATED
11:11:58.471 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:11:58.474 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.200.188:45753(127.20.200.188:45753)] handling channelInactive
11:11:58.474 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.200.188:45753(127.20.200.188:45753)] cleaning up while in state READY due to: connection closed
11:11:58.478 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:11:58.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:11:58.485937 21282 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 21732
11:11:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T11:11:59Z chronyd exiting
11:12:00.800 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:12:00.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:12:00.802085 20530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 20817
11:12:00.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:12:00.835839 20530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 20995
11:12:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:12:00.867636 20530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 21137
11:12:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:12:00.904409 20530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 20543
11:12:00.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:12:00.945765 20530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 20614
11:12:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260807 11:12:00.987553 20530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskV256yr/build/tsan/bin/kudu with pid 20691
11:12:02.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-07T11:12:02Z chronyd exiting
11:12:03.272 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29908), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [634ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [639ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [667ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [671ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [673ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [680ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [681ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [688ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [702ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [710ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [716ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [725ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [740ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [745ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [781ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [787ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [789ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [821ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [827ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [829ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [960ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [966ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [967ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1237ms] refreshing cache from master, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1246ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1617ms] refreshing cache from master, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1625ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1627ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1627ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2202ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2204ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2637ms] refreshing cache from master, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2644ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2645ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [3537ms] refreshing cache from master, [3537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [3539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [3540ms] too many traces: truncated at 100 traces, deferred=Deferred@403535664(state=PENDING, result=null, callback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060), errback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.200.190:44227
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29908), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [634ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [639ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [667ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [671ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [673ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [680ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [681ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [688ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [702ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [710ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [716ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [725ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [740ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [745ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [781ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [787ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [789ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [821ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [827ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [829ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [960ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [966ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [967ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1237ms] refreshing cache from master, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1246ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1617ms] refreshing cache from master, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1625ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1627ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1627ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2202ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2204ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2637ms] refreshing cache from master, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2644ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2645ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [3537ms] refreshing cache from master, [3537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [3539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [3540ms] too many traces: truncated at 100 traces, deferred=Deferred@403535664(state=PENDING, result=null, callback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060), errback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.200.190:44227
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
11:12:03.829 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
11:12:03.830 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29908), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [634ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [639ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [667ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [671ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [673ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [680ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [681ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [688ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [702ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [710ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [716ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [725ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [740ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [745ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [781ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [787ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [789ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [821ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [827ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [829ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [960ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [966ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [967ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1237ms] refreshing cache from master, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1246ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1617ms] refreshing cache from master, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1625ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1627ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1627ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2202ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2204ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2637ms] refreshing cache from master, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2644ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2645ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [3537ms] refreshing cache from master, [3537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [3539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [3540ms] too many traces: truncated at 100 traces, deferred=Deferred@403535664(state=PENDING, result=null, callback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060), errback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.200.190:44227
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29908), Traces: [0ms] refreshing cache from master, [35ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [126ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [194ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [634ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [639ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [663ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [667ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [671ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [672ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [673ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [680ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [681ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [681ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [688ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [689ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [702ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [702ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [710ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [716ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [725ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [737ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [740ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [740ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [743ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [745ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [781ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [787ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [789ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [821ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [827ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [829ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [957ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [960ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [966ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [967ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1237ms] refreshing cache from master, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1241ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1243ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1246ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1617ms] refreshing cache from master, [1617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [1618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [1621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [1621ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [1625ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [1627ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [1627ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2197ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2202ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2204ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2637ms] refreshing cache from master, [2637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [2640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.189:42931: Network error: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.190:44227: Network error: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [2644ms] Sub RPC ConnectToMaster: received response from server master-127.20.200.188:45753: OK, [2645ms] delaying RPC due to: Service unavailable: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227, [3537ms] refreshing cache from master, [3537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.189:42931, [3538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.190:44227, [3539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.200.188:45753, [3540ms] too many traces: truncated at 100 traces, deferred=Deferred@403535664(state=PENDING, result=null, callback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060), errback=(continuation of Deferred@1265971176 after retry RPC after error@1802939249) -> (continuation of Deferred@1727539644 after retry RPC after error@364865188) -> (continuation of Deferred@2010583444 after retry RPC after error@690225468) -> (continuation of Deferred@238450797 after retry RPC after error@1624417609) -> (continuation of Deferred@801526646 after retry RPC after error@652380755) -> (continuation of Deferred@455769074 after retry RPC after error@1856470323) -> (continuation of Deferred@1936186994 after retry RPC after error@1097167455) -> (continuation of Deferred@156791914 after retry RPC after error@1556340375) -> (continuation of Deferred@286924595 after retry RPC after error@2064085189) -> (continuation of Deferred@825663723 after retry RPC after error@1942218556) -> (continuation of Deferred@845254128 after retry RPC after error@1923109409) -> (continuation of Deferred@1989514598 after retry RPC after error@436430479) -> (continuation of Deferred@607348274 after retry RPC after error@1416759759) -> (continuation of Deferred@244750466 after retry RPC after error@262185153) -> (continuation of Deferred@807224817 after retry RPC after error@1961168460) -> (continuation of Deferred@1317915354 after retry RPC after error@1151458658) -> (continuation of Deferred@508111562 after retry RPC after error@1107476513) -> (continuation of Deferred@1652099489 after retry RPC after error@679055937) -> (continuation of Deferred@2142364414 after retry RPC after error@736503521) -> (continuation of Deferred@33035837 after retry RPC after error@1702324185) -> (continuation of Deferred@354541467 after retry RPC after error@267006704) -> (continuation of Deferred@1315083470 after retry RPC after error@284523380) -> (continuation of Deferred@196326390 after retry RPC after error@1427144130) -> (continuation of Deferred@260982602 after retry RPC after error@882615060)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.20.200.189:42931,127.20.200.190:44227,127.20.200.188:45753) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.189:42931(127.20.200.189:42931): Connection refused: /127.20.200.189:42931,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.20.200.190:44227(127.20.200.190:44227): Connection refused: /127.20.200.190:44227
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.20.200.190:44227
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)