14:14:38.768 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:14:38.768 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLlQHbR/test-tmp
14:14:38.768 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:14:38.768 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLlQHbR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:14:38.771 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:14:39.114 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:14:39.114 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:14:39.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.117571 10848 mini_oidc.cc:150] Starting JWKS server
14:14:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.121176 10848 webserver.cc:492] Webserver started at http://127.0.0.1:42835/ using document root <none> and password file <none>
14:14:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.121644 10848 mini_oidc.cc:186] Starting OIDC Discovery server
14:14:39.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.123627 10848 webserver.cc:492] Webserver started at http://127.0.0.1:39855/ using document root <none> and password file <none>
14:14:39.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:14:39Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:14:39.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:14:39Z Disabled control of system clock
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.189797 10848 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/wal
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/logs
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data/info.pb
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.62:41699
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.62
14:14:39.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:39.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42835/jwks.json with env {}
14:14:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.501901 10860 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:14:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.502475 10860 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.502956 10860 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:14:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.536895 10860 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:14:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.537205 10860 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:14:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.537403 10860 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.537602 10860 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:14:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.537796 10860 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.537953 10860 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42835/jwks.json
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.576691 10860 master_runner.cc:386] Master server non-default flags:
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/wal
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.62:41699
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42835/jwks.json
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data/info.pb
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.62
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/logs
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:14:39.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:39.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.578042 10860 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:39.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:39.579815 10860 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.591320 10866 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:40.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:39.592705 10867 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:40.891775 10869 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:40.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:40.894536 10868 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1297 milliseconds
14:14:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:40.894856 10860 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.303s user 0.363s sys 0.932s
14:14:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:40.895282 10860 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.304s user 0.363s sys 0.932s
14:14:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.895602 10860 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:40.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.897245 10860 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.900605 10860 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:40.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.902241 10860 hybrid_clock.cc:648] HybridClock initialized: now 1777904080902197 us; error 41 us; skew 500 ppm
14:14:40.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.903524 10860 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:40.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.912767 10860 webserver.cc:492] Webserver started at http://127.10.152.62:34997/ using document root <none> and password file <none>
14:14:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.914242 10860 fs_manager.cc:362] Metadata directory not provided
14:14:40.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.914569 10860 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:40.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.915190 10860 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:14:40.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.922621 10860 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data/instance:
14:14:40.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "890caf187d5b4b808f8f6e8463ae5e0a"
14:14:40.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:40 on dist-test-slave-vb5z"
14:14:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.924371 10860 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/wal/instance:
14:14:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "890caf187d5b4b808f8f6e8463ae5e0a"
14:14:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:40 on dist-test-slave-vb5z"
14:14:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.934939 10860 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.010s sys 0.000s
14:14:40.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.943594 10876 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:40.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.945367 10860 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
14:14:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.945961 10860 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/wal
14:14:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "890caf187d5b4b808f8f6e8463ae5e0a"
14:14:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:40 on dist-test-slave-vb5z"
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:40.946609 10860 fs_report.cc:389] FS layout report
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/wal
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/wal
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data/data
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:41.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.036384 10860 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:41.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.037902 10860 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:41.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.038324 10860 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.112999 10860 rpc_server.cc:307] RPC server started. Bound to: 127.10.152.62:41699
14:14:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.113116 10927 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.152.62:41699 every 8 connection(s)
14:14:41.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.115708 10860 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/data/info.pb
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.118022 10848 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 10860
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.118582 10848 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-0/wal/instance
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.122321 10928 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.122309 10848 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/wal
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/logs
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data/info.pb
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:41.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.61:44711
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.61
14:14:41.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42835/jwks.json with env {}
14:14:41.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.134353 10928 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } has no permanent_uuid. Determining permanent_uuid...
14:14:41.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.163647 10928 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } has no permanent_uuid. Determining permanent_uuid...
14:14:41.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.166440 10879 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:41.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.170722 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.61:44711: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:41.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.220219 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } attempt: 1
14:14:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.224707 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.61:44711: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:41.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.293263 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } attempt: 2
14:14:41.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.297542 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.61:44711: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:41.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.389039 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } attempt: 3
14:14:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.393128 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.61:44711: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:41.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.439106 10930 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:14:41.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.439678 10930 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:41.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.440135 10930 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:14:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.473239 10930 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:14:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.473526 10930 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:14:41.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.473729 10930 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:41.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.474048 10930 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:14:41.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.474328 10930 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:41.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.474493 10930 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42835/jwks.json
14:14:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.511483 10930 master_runner.cc:386] Master server non-default flags:
14:14:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/wal
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.61:44711
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42835/jwks.json
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data/info.pb
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.61
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/logs
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.512995 10930 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.514658 10930 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:41.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.526062 10939 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.536625 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } attempt: 4
14:14:41.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.544252 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.61:44711: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:41.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:41.843822 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } attempt: 5
14:14:41.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.852774 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.61:44711: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:42.400418 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } attempt: 6
14:14:42.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:42.425830 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.61:44711: Network error: Client connection negotiation failed: client connection to 127.10.152.61:44711: connect: Connection refused (error 111)
14:14:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:42.932235 10938 debug-util.cc:398] Leaking SignalData structure 0x7b0800034cc0 after lost signal to thread 10930
14:14:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:41.527432 10940 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:43.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.130896 10930 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.605s user 0.605s sys 0.984s
14:14:43.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.131929 10930 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.606s user 0.606s sys 0.984s
14:14:43.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.133333 10942 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:43.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.137684 10941 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1609 milliseconds
14:14:43.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.137684 10930 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:43.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.139580 10930 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:43.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.142052 10930 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:43.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.143560 10930 hybrid_clock.cc:648] HybridClock initialized: now 1777904083143515 us; error 37 us; skew 500 ppm
14:14:43.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.144414 10930 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:43.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.150646 10930 webserver.cc:492] Webserver started at http://127.10.152.61:44945/ using document root <none> and password file <none>
14:14:43.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.151803 10930 fs_manager.cc:362] Metadata directory not provided
14:14:43.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.152062 10930 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:43.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.152566 10930 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:14:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.157044 10930 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data/instance:
14:14:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53fe6be5c84f43f8ae5634908915774e"
14:14:43.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:43 on dist-test-slave-vb5z"
14:14:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.158265 10930 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/wal/instance:
14:14:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53fe6be5c84f43f8ae5634908915774e"
14:14:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:43 on dist-test-slave-vb5z"
14:14:43.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.165621 10930 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
14:14:43.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.171561 10951 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.172784 10930 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
14:14:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.173189 10930 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/wal
14:14:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "53fe6be5c84f43f8ae5634908915774e"
14:14:43.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:43 on dist-test-slave-vb5z"
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.173552 10930 fs_report.cc:389] FS layout report
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/wal
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/wal
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data/data
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:43.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.235384 10930 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.237035 10930 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:43.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.237564 10930 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:43.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.312376 10930 rpc_server.cc:307] RPC server started. Bound to: 127.10.152.61:44711
14:14:43.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.312541 11002 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.152.61:44711 every 8 connection(s)
14:14:43.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.315306 10930 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/data/info.pb
14:14:43.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.321239 11003 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.325769 10848 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 10930
14:14:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.326256 10848 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-1/wal/instance
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.329574 10848 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/wal
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/logs
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data/info.pb
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.60:39017
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.60
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:43.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42835/jwks.json with env {}
14:14:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.337615 11003 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } has no permanent_uuid. Determining permanent_uuid...
14:14:43.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.364449 11003 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } has no permanent_uuid. Determining permanent_uuid...
14:14:43.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.380219 11003 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } has no permanent_uuid. Determining permanent_uuid...
14:14:43.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.382781 10955 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.387516 11003 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.437078 11003 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 1
14:14:43.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.442233 11003 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.486366 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } attempt: 7
14:14:43.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.498876 10928 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } has no permanent_uuid. Determining permanent_uuid...
14:14:43.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.504969 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.510658 11003 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 2
14:14:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.515223 11003 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.563519 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 1
14:14:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.567938 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.606704 11003 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 3
14:14:43.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.611851 11003 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.649430 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 2
14:14:43.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.653589 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.667220 11005 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:14:43.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.667855 11005 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:43.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.668346 11005 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.701179 11005 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.701488 11005 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:14:43.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.701682 11005 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.701893 11005 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.702090 11005 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:43.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.702246 11005 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42835/jwks.json
14:14:43.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.739082 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 3
14:14:43.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.742313 11005 master_runner.cc:386] Master server non-default flags:
14:14:43.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/wal
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.60:39017
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42835/jwks.json
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data/info.pb
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.60
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/logs
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.744271 11005 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:43.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.744532 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.746148 11005 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.755333 11003 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 4
14:14:43.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.757906 11017 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:43.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.762039 11003 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:43.885017 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 4
14:14:43.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.892138 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:44.061587 11003 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 5
14:14:44.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:44.068771 11003 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:44.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:44.175815 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 5
14:14:44.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:44.183413 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:44.616441 11003 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 6
14:14:44.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:44.629629 11003 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:44.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:44.736136 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 6
14:14:44.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:44.748045 10928 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.10.152.60:39017: Network error: Client connection negotiation failed: client connection to 127.10.152.60:39017: connect: Connection refused (error 111)
14:14:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.762962 11020 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:43.760277 11018 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:45.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:45.014997 11019 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1250 milliseconds
14:14:45.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.015122 11005 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.016454 11005 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.019645 11005 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.021126 11005 hybrid_clock.cc:648] HybridClock initialized: now 1777904085021068 us; error 74 us; skew 500 ppm
14:14:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.021909 11005 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.028355 11005 webserver.cc:492] Webserver started at http://127.10.152.60:44497/ using document root <none> and password file <none>
14:14:45.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.029261 11005 fs_manager.cc:362] Metadata directory not provided
14:14:45.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.029454 11005 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.029855 11005 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:14:45.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.034318 11005 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data/instance:
14:14:45.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a26333a2a9874c849060bb7b95496f4f"
14:14:45.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:45 on dist-test-slave-vb5z"
14:14:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.035393 11005 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/wal/instance:
14:14:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a26333a2a9874c849060bb7b95496f4f"
14:14:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:45 on dist-test-slave-vb5z"
14:14:45.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.042590 11005 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
14:14:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.048586 11031 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.049739 11005 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:14:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.050123 11005 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/wal
14:14:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a26333a2a9874c849060bb7b95496f4f"
14:14:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:45 on dist-test-slave-vb5z"
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.050638 11005 fs_report.cc:389] FS layout report
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/wal
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/wal
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data/data
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:45.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.114113 11005 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:45.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.115738 11005 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:45.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.116254 11005 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:45.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.190519 11005 rpc_server.cc:307] RPC server started. Bound to: 127.10.152.60:39017
14:14:45.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.190587 11082 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.152.60:39017 every 8 connection(s)
14:14:45.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.193271 11005 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/data/info.pb
14:14:45.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.199101 11083 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.200348 10848 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11005
14:14:45.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.200670 10848 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/master-2/wal/instance
14:14:45.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.214447 11083 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } has no permanent_uuid. Determining permanent_uuid...
14:14:45.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.242631 11083 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } has no permanent_uuid. Determining permanent_uuid...
14:14:45.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.254501 11083 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } has no permanent_uuid. Determining permanent_uuid...
14:14:45.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.271727 11083 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f: Bootstrap starting.
14:14:45.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.278050 11083 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f: Neither blocks nor log segments found. Creating new log.
14:14:45.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.279775 11083 log.cc:826] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f: Log is configured to *not* fsync() on all Append() calls
14:14:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.284835 11083 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f: No bootstrap required, opened a new log
14:14:45.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.305193 11083 raft_consensus.cc:359] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.305831 11083 raft_consensus.cc:385] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:45.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.306035 11083 raft_consensus.cc:740] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a26333a2a9874c849060bb7b95496f4f, State: Initialized, Role: FOLLOWER
14:14:45.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.306764 11083 consensus_queue.cc:260] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.309111 11092 sys_catalog.cc:455] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:45.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.310024 11092 sys_catalog.cc:458] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: This master's current role is: FOLLOWER
14:14:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.311007 11083 sys_catalog.cc:565] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: configured and running, proceeding with master startup.
14:14:45.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:45.348094 11103 catalog_manager.cc:1568] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:14:45.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:45.348472 11103 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:14:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.344378 11005 master_runner.cc:186] Error getting master registration for 127.10.152.62:41699: OK, instance_id { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" instance_seqno: 1777904081092780 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:14:45.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.368299 11005 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:45.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:45.371572 11037 tablet.cc:2404] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:14:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.510416 11092 raft_consensus.cc:493] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:14:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.510933 11092 raft_consensus.cc:515] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.513458 11092 leader_election.cc:290] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 890caf187d5b4b808f8f6e8463ae5e0a (127.10.152.62:41699), 53fe6be5c84f43f8ae5634908915774e (127.10.152.61:44711)
14:14:45.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.690181 11003 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 7
14:14:45.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.714169 11003 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e: Bootstrap starting.
14:14:45.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.721733 11003 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e: Neither blocks nor log segments found. Creating new log.
14:14:45.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.724200 11003 log.cc:826] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e: Log is configured to *not* fsync() on all Append() calls
14:14:45.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.729789 11003 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e: No bootstrap required, opened a new log
14:14:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.758324 11003 raft_consensus.cc:359] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.759210 11003 raft_consensus.cc:385] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:45.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.759528 11003 raft_consensus.cc:740] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53fe6be5c84f43f8ae5634908915774e, State: Initialized, Role: FOLLOWER
14:14:45.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.760462 11003 consensus_queue.cc:260] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.763422 11107 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.764895 11107 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [sys.catalog]: This master's current role is: FOLLOWER
14:14:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.765444 11003 sys_catalog.cc:565] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [sys.catalog]: configured and running, proceeding with master startup.
14:14:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.765630 10978 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a26333a2a9874c849060bb7b95496f4f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53fe6be5c84f43f8ae5634908915774e" is_pre_election: true
14:14:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.766498 10978 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a26333a2a9874c849060bb7b95496f4f in term 0.
14:14:45.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.768091 11034 leader_election.cc:304] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 53fe6be5c84f43f8ae5634908915774e, a26333a2a9874c849060bb7b95496f4f; no voters:
14:14:45.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.769201 11092 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Leader pre-election won for term 1
14:14:45.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.769563 11092 raft_consensus.cc:493] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:14:45.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.769901 11092 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 0 FOLLOWER]: Advancing to term 1
14:14:45.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.780539 11092 raft_consensus.cc:515] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.781802 10928 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } attempt: 7
14:14:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.785234 10978 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a26333a2a9874c849060bb7b95496f4f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53fe6be5c84f43f8ae5634908915774e"
14:14:45.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.785866 10978 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [term 0 FOLLOWER]: Advancing to term 1
14:14:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.788108 11092 leader_election.cc:290] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [CANDIDATE]: Term 1 election: Requested vote from peers 890caf187d5b4b808f8f6e8463ae5e0a (127.10.152.62:41699), 53fe6be5c84f43f8ae5634908915774e (127.10.152.61:44711)
14:14:45.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.793916 10978 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a26333a2a9874c849060bb7b95496f4f in term 1.
14:14:45.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.795848 11034 leader_election.cc:304] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 53fe6be5c84f43f8ae5634908915774e, a26333a2a9874c849060bb7b95496f4f; no voters:
14:14:45.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.796916 11092 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 1 FOLLOWER]: Leader election won for term 1
14:14:45.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.799645 11092 raft_consensus.cc:697] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [term 1 LEADER]: Becoming Leader. State: Replica: a26333a2a9874c849060bb7b95496f4f, State: Running, Role: LEADER
14:14:45.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.800697 11092 consensus_queue.cc:237] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.814170 11113 sys_catalog.cc:455] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: SysCatalogTable state changed. Reason: New leader a26333a2a9874c849060bb7b95496f4f. Latest consensus state: current_term: 1 leader_uuid: "a26333a2a9874c849060bb7b95496f4f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:45.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.814940 11113 sys_catalog.cc:458] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: This master's current role is: LEADER
14:14:45.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.830468 11117 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
14:14:45.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.838706 11117 catalog_manager.cc:1494] Initializing Kudu cluster ID...
14:14:45.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.839422 10928 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a: Bootstrap starting.
14:14:45.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.856580 11123 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:14:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.861515 10978 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [term 1 FOLLOWER]: Refusing update from remote peer a26333a2a9874c849060bb7b95496f4f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:14:45.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.876891 10928 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a: Neither blocks nor log segments found. Creating new log.
14:14:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.879494 10928 log.cc:826] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a: Log is configured to *not* fsync() on all Append() calls
14:14:45.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.868417 11113 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.011s
14:14:45.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.910687 10928 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a: No bootstrap required, opened a new log
14:14:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.865149 10930 master_runner.cc:186] Error getting master registration for 127.10.152.62:41699: OK, instance_id { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" instance_seqno: 1777904081092780 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:14:45.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.920928 10930 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:14:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.923439 11107 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [sys.catalog]: SysCatalogTable state changed. Reason: New leader a26333a2a9874c849060bb7b95496f4f. Latest consensus state: current_term: 1 leader_uuid: "a26333a2a9874c849060bb7b95496f4f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.924216 11107 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [sys.catalog]: This master's current role is: FOLLOWER
14:14:45.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.933692 11125 mvcc.cc:204] Tried to move back new op lower bound from 7282295135672713216 to 7282295135450247168. Current Snapshot: MvccSnapshot[applied={T|T < 7282295135672713216}]
14:14:45.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.938768 11113 sys_catalog.cc:455] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a26333a2a9874c849060bb7b95496f4f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:45.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.939800 11113 sys_catalog.cc:458] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: This master's current role is: LEADER
14:14:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.941318 11107 sys_catalog.cc:455] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a26333a2a9874c849060bb7b95496f4f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.941998 11107 sys_catalog.cc:458] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e [sys.catalog]: This master's current role is: FOLLOWER
14:14:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.942603 11117 catalog_manager.cc:1357] Generated new cluster ID: 5d0a97284cb3402bac4a632ade146706
14:14:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.942896 11117 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
14:14:45.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.944381 10928 raft_consensus.cc:359] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.945238 10928 raft_consensus.cc:385] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:14:45.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.945561 10928 raft_consensus.cc:740] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 890caf187d5b4b808f8f6e8463ae5e0a, State: Initialized, Role: FOLLOWER
14:14:45.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.946481 10928 consensus_queue.cc:260] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } }
14:14:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.954802 11129 sys_catalog.cc:455] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.955636 11129 sys_catalog.cc:458] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [sys.catalog]: This master's current role is: FOLLOWER
14:14:45.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.958626 10928 sys_catalog.cc:565] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [sys.catalog]: configured and running, proceeding with master startup.
14:14:45.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.962528 10902 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a26333a2a9874c849060bb7b95496f4f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "890caf187d5b4b808f8f6e8463ae5e0a"
14:14:45.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.963603 10902 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [term 0 FOLLOWER]: Advancing to term 1
14:14:45.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.966413 10903 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a26333a2a9874c849060bb7b95496f4f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" is_pre_election: true
14:14:45.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.977872 10902 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a26333a2a9874c849060bb7b95496f4f in term 1.
14:14:45.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.978742 11117 catalog_manager.cc:1380] Generated new certificate authority record
14:14:45.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.979391 10901 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [term 1 FOLLOWER]: Refusing update from remote peer a26333a2a9874c849060bb7b95496f4f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:14:45.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.980793 11117 catalog_manager.cc:1514] Loading token signing keys...
14:14:45.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:45.987645 11113 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [LEADER]: Connected to new peer: Peer: permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:14:46.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.019325 11117 catalog_manager.cc:6044] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f: Generated new TSK 0
14:14:46.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.020556 11117 catalog_manager.cc:1524] Initializing in-progress tserver states...
14:14:46.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.039976 11092 sys_catalog.cc:455] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a26333a2a9874c849060bb7b95496f4f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:46.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.040802 11092 sys_catalog.cc:458] T 00000000000000000000000000000000 P a26333a2a9874c849060bb7b95496f4f [sys.catalog]: This master's current role is: LEADER
14:14:46.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.041283 11129 sys_catalog.cc:455] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [sys.catalog]: SysCatalogTable state changed. Reason: New leader a26333a2a9874c849060bb7b95496f4f. Latest consensus state: current_term: 1 leader_uuid: "a26333a2a9874c849060bb7b95496f4f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:46.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.042032 11129 sys_catalog.cc:458] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [sys.catalog]: This master's current role is: FOLLOWER
14:14:46.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.041810 11142 sys_catalog.cc:455] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a26333a2a9874c849060bb7b95496f4f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "890caf187d5b4b808f8f6e8463ae5e0a" member_type: VOTER last_known_addr { host: "127.10.152.62" port: 41699 } } peers { permanent_uuid: "53fe6be5c84f43f8ae5634908915774e" member_type: VOTER last_known_addr { host: "127.10.152.61" port: 44711 } } peers { permanent_uuid: "a26333a2a9874c849060bb7b95496f4f" member_type: VOTER last_known_addr { host: "127.10.152.60" port: 39017 } } }
14:14:46.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.042591 11142 sys_catalog.cc:458] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a [sys.catalog]: This master's current role is: FOLLOWER
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.057819 10848 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/wal
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/logs
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data/info.pb
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.1:0
14:14:46.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.152.1
14:14:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.1
14:14:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:46.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:14:46.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.061650 11145 catalog_manager.cc:1269] Loaded cluster ID: 5d0a97284cb3402bac4a632ade146706
14:14:46.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.062165 11145 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a: loading cluster ID for follower catalog manager: success
14:14:46.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.070096 11145 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a: acquiring CA information for follower catalog manager: success
14:14:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.074738 11145 catalog_manager.cc:883] Not found: no TSK found in the system table: failed to prepare follower catalog manager, will retry
14:14:46.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.496034 11147 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:14:46.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.496613 11147 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:46.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.497153 11147 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:14:46.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.535966 11147 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:14:46.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.536907 11147 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.152.1
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.599833 11147 tablet_server_runner.cc:78] Tablet server non-default flags:
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/wal
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.1:0
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data/info.pb
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.1
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.152.1
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/logs
14:14:46.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:46.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.601608 11147 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:46.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.603875 11147 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:46.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.622932 11153 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:46.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.887751 11123 catalog_manager.cc:1269] Loaded cluster ID: 5d0a97284cb3402bac4a632ade146706
14:14:46.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.888302 11123 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e: loading cluster ID for follower catalog manager: success
14:14:46.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.920511 11123 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e: acquiring CA information for follower catalog manager: success
14:14:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:46.927215 11123 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 53fe6be5c84f43f8ae5634908915774e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:47.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:47.082564 11145 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 890caf187d5b4b808f8f6e8463ae5e0a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:14:49.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:48.021271 11152 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 11147
14:14:49.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:49.357692 11147 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.718s user 0.000s sys 0.001s
14:14:49.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:49.360039 11147 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.720s user 0.000s sys 0.002s
14:14:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:46.629261 11154 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:49.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:49.362244 11157 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:49.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.368695 11147 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:49.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:49.369244 11155 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2738 milliseconds
14:14:49.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.370728 11147 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:49.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.381027 11147 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.382488 11147 hybrid_clock.cc:648] HybridClock initialized: now 1777904089382415 us; error 86 us; skew 500 ppm
14:14:49.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.383733 11147 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.405537 11147 webserver.cc:492] Webserver started at http://127.10.152.1:36325/ using document root <none> and password file <none>
14:14:49.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.407081 11147 fs_manager.cc:362] Metadata directory not provided
14:14:49.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.407380 11147 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:49.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.408099 11147 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:14:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.414942 11147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data/instance:
14:14:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e9433033c8a453e892e23db06560c29"
14:14:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:49 on dist-test-slave-vb5z"
14:14:49.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.416586 11147 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/wal/instance:
14:14:49.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e9433033c8a453e892e23db06560c29"
14:14:49.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:49 on dist-test-slave-vb5z"
14:14:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.426573 11147 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
14:14:49.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.434945 11165 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.436429 11147 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
14:14:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.436872 11147 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/wal
14:14:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2e9433033c8a453e892e23db06560c29"
14:14:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:49 on dist-test-slave-vb5z"
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.437325 11147 fs_report.cc:389] FS layout report
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/wal
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/wal
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data/data
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:49.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.538755 11147 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.540875 11147 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:49.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.541451 11147 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:49.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.545130 11147 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:14:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.551414 11147 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:14:49.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.551767 11147 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.001s
14:14:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.552075 11147 ts_tablet_manager.cc:616] Registered 0 tablets
14:14:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.552306 11147 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:14:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.744632 11147 rpc_server.cc:307] RPC server started. Bound to: 127.10.152.1:35767
14:14:49.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.744761 11278 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.152.1:35767 every 8 connection(s)
14:14:49.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.748368 11147 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/data/info.pb
14:14:49.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.756349 10848 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11147
14:14:49.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.756985 10848 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-0/wal/instance
14:14:49.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.799438 10848 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/wal
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/logs
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data/info.pb
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.2:0
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.152.2
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.2
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:49.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:14:49.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.817989 11280 heartbeater.cc:344] Connected to a master server at 127.10.152.61:44711
14:14:49.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.820475 11280 heartbeater.cc:461] Registering TS with master...
14:14:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.821929 11280 heartbeater.cc:507] Master 127.10.152.61:44711 requested a full tablet report, sending...
14:14:49.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.825611 10968 ts_manager.cc:194] Registered new tserver with Master: 2e9433033c8a453e892e23db06560c29 (127.10.152.1:35767)
14:14:49.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.827375 11279 heartbeater.cc:344] Connected to a master server at 127.10.152.60:39017
14:14:49.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.827802 11279 heartbeater.cc:461] Registering TS with master...
14:14:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.828735 11279 heartbeater.cc:507] Master 127.10.152.60:39017 requested a full tablet report, sending...
14:14:49.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.831282 11048 ts_manager.cc:194] Registered new tserver with Master: 2e9433033c8a453e892e23db06560c29 (127.10.152.1:35767)
14:14:49.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.833948 11048 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.152.1:50937
14:14:49.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.855352 11283 heartbeater.cc:344] Connected to a master server at 127.10.152.62:41699
14:14:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.855810 11283 heartbeater.cc:461] Registering TS with master...
14:14:49.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.856539 11283 heartbeater.cc:507] Master 127.10.152.62:41699 requested a full tablet report, sending...
14:14:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:49.860119 10893 ts_manager.cc:194] Registered new tserver with Master: 2e9433033c8a453e892e23db06560c29 (127.10.152.1:35767)
14:14:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:50.290601 11289 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:14:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:50.291201 11289 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:50.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:50.291780 11289 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:14:50.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:50.331368 11289 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:14:50.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:50.332386 11289 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.152.2
14:14:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:50.377216 11289 tablet_server_runner.cc:78] Tablet server non-default flags:
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/wal
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:50.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.2:0
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data/info.pb
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.2
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.152.2
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/logs
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:50.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:50.378962 11289 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:50.381219 11289 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:50.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:50.396646 11295 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:50.842859 11279 heartbeater.cc:499] Master 127.10.152.60:39017 was elected leader, sending a full tablet report...
14:14:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:51.799362 11294 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 11289
14:14:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.108034 11294 kernel_stack_watchdog.cc:198] Thread 11289 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
14:14:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:14:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:14:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:14:52.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:14:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.108693 11289 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.710s user 0.558s sys 1.138s
14:14:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:50.398955 11296 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.109102 11289 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.711s user 0.558s sys 1.138s
14:14:52.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.111534 11298 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:52.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.114598 11297 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1710 milliseconds
14:14:52.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.115353 11289 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:52.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.116524 11289 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.118644 11289 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:52.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.120116 11289 hybrid_clock.cc:648] HybridClock initialized: now 1777904092120082 us; error 47 us; skew 500 ppm
14:14:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.121024 11289 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:52.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.127837 11289 webserver.cc:492] Webserver started at http://127.10.152.2:45739/ using document root <none> and password file <none>
14:14:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.128857 11289 fs_manager.cc:362] Metadata directory not provided
14:14:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.129119 11289 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:52.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.129695 11289 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:14:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.134512 11289 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data/instance:
14:14:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5bfb850afce345ad8c8f35f0ea656cd7"
14:14:52.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:52 on dist-test-slave-vb5z"
14:14:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.135908 11289 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/wal/instance:
14:14:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5bfb850afce345ad8c8f35f0ea656cd7"
14:14:52.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:52 on dist-test-slave-vb5z"
14:14:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.143599 11289 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
14:14:52.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.149875 11306 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:52.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.151135 11289 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:14:52.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.151592 11289 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/wal
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5bfb850afce345ad8c8f35f0ea656cd7"
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:52 on dist-test-slave-vb5z"
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.151985 11289 fs_report.cc:389] FS layout report
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/wal
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/wal
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data/data
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:52.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:52.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.212545 11289 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:52.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.214296 11289 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:52.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.214830 11289 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:52.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.217696 11289 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:14:52.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.222177 11289 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:14:52.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.222450 11289 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:14:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.222836 11289 ts_tablet_manager.cc:616] Registered 0 tablets
14:14:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.223047 11289 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:14:52.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.373029 11289 rpc_server.cc:307] RPC server started. Bound to: 127.10.152.2:46737
14:14:52.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.373147 11418 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.152.2:46737 every 8 connection(s)
14:14:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.375782 11289 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/data/info.pb
14:14:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.377573 10848 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11289
14:14:52.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.378173 10848 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-1/wal/instance
14:14:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.416322 10848 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/wal
14:14:52.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/logs
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data/info.pb
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:52.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.3:0
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.152.3
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.3
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:52.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:14:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.427641 11421 heartbeater.cc:344] Connected to a master server at 127.10.152.62:41699
14:14:52.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.428184 11421 heartbeater.cc:461] Registering TS with master...
14:14:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.429556 11421 heartbeater.cc:507] Master 127.10.152.62:41699 requested a full tablet report, sending...
14:14:52.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.433015 10893 ts_manager.cc:194] Registered new tserver with Master: 5bfb850afce345ad8c8f35f0ea656cd7 (127.10.152.2:46737)
14:14:52.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.433190 11420 heartbeater.cc:344] Connected to a master server at 127.10.152.61:44711
14:14:52.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.433458 11419 heartbeater.cc:344] Connected to a master server at 127.10.152.60:39017
14:14:52.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.433584 11420 heartbeater.cc:461] Registering TS with master...
14:14:52.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.433845 11419 heartbeater.cc:461] Registering TS with master...
14:14:52.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.434583 11420 heartbeater.cc:507] Master 127.10.152.61:44711 requested a full tablet report, sending...
14:14:52.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.434762 11419 heartbeater.cc:507] Master 127.10.152.60:39017 requested a full tablet report, sending...
14:14:52.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.437471 11048 ts_manager.cc:194] Registered new tserver with Master: 5bfb850afce345ad8c8f35f0ea656cd7 (127.10.152.2:46737)
14:14:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.437958 10968 ts_manager.cc:194] Registered new tserver with Master: 5bfb850afce345ad8c8f35f0ea656cd7 (127.10.152.2:46737)
14:14:52.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.439198 11048 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.152.2:59851
14:14:52.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.735383 11429 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:14:52.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.736076 11429 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:52.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.736552 11429 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:14:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.778234 11429 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:14:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.779073 11429 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.152.3
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.817765 11429 tablet_server_runner.cc:78] Tablet server non-default flags:
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.10.152.20:36167
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/wal
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.10.152.3:0
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data/info.pb
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.10.152.3
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.10.152.62:41699,127.10.152.61:44711,127.10.152.60:39017
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.10.152.3
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/logs
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:52.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.819113 11429 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:52.820755 11429 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:52.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.833858 11435 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:53.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:53.443856 11419 heartbeater.cc:499] Master 127.10.152.60:39017 was elected leader, sending a full tablet report...
14:14:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:54.237548 11434 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 11429
14:14:54.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:54.615816 11429 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.782s user 0.648s sys 1.127s
14:14:54.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:52.835001 11436 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:54.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:54.616227 11429 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.783s user 0.648s sys 1.127s
14:14:54.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:54.618963 11438 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:54.620779 11437 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1782 milliseconds
14:14:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.620788 11429 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.622100 11429 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:54.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.624301 11429 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:54.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.625661 11429 hybrid_clock.cc:648] HybridClock initialized: now 1777904094625634 us; error 57 us; skew 500 ppm
14:14:54.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.626471 11429 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:54.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.633039 11429 webserver.cc:492] Webserver started at http://127.10.152.3:34447/ using document root <none> and password file <none>
14:14:54.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.633934 11429 fs_manager.cc:362] Metadata directory not provided
14:14:54.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.634133 11429 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:54.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.634526 11429 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:14:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.638882 11429 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data/instance:
14:14:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eeb9132d8364461bb4446d584eb2da3e"
14:14:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:54 on dist-test-slave-vb5z"
14:14:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.640156 11429 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/wal/instance:
14:14:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eeb9132d8364461bb4446d584eb2da3e"
14:14:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:54 on dist-test-slave-vb5z"
14:14:54.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.647652 11429 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
14:14:54.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.653461 11446 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:54.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.654659 11429 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.654960 11429 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/wal
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eeb9132d8364461bb4446d584eb2da3e"
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:54 on dist-test-slave-vb5z"
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.655275 11429 fs_report.cc:389] FS layout report
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/wal
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/wal
14:14:54.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data/data
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:54.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.709075 11429 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.710793 11429 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.711320 11429 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.714186 11429 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:14:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.718521 11429 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:14:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.718812 11429 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:14:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.719165 11429 ts_tablet_manager.cc:616] Registered 0 tablets
14:14:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.719350 11429 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:14:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.869256 11429 rpc_server.cc:307] RPC server started. Bound to: 127.10.152.3:40657
14:14:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.869390 11558 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.10.152.3:40657 every 8 connection(s)
14:14:54.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.872133 11429 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/data/info.pb
14:14:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.874950 10848 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11429
14:14:54.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.875382 10848 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster5005910678456125811/ts-2/wal/instance
14:14:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.923545 11559 heartbeater.cc:344] Connected to a master server at 127.10.152.60:39017
14:14:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.924010 11559 heartbeater.cc:461] Registering TS with master...
14:14:54.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.924757 11561 heartbeater.cc:344] Connected to a master server at 127.10.152.62:41699
14:14:54.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.924767 11560 heartbeater.cc:344] Connected to a master server at 127.10.152.61:44711
14:14:54.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.925235 11560 heartbeater.cc:461] Registering TS with master...
14:14:54.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.925402 11561 heartbeater.cc:461] Registering TS with master...
14:14:54.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.925489 11559 heartbeater.cc:507] Master 127.10.152.60:39017 requested a full tablet report, sending...
14:14:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.926398 11560 heartbeater.cc:507] Master 127.10.152.61:44711 requested a full tablet report, sending...
14:14:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.926409 11561 heartbeater.cc:507] Master 127.10.152.62:41699 requested a full tablet report, sending...
14:14:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.928668 11048 ts_manager.cc:194] Registered new tserver with Master: eeb9132d8364461bb4446d584eb2da3e (127.10.152.3:40657)
14:14:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.929333 10968 ts_manager.cc:194] Registered new tserver with Master: eeb9132d8364461bb4446d584eb2da3e (127.10.152.3:40657)
14:14:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.930605 11048 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.10.152.3:48339
14:14:54.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.934384 10893 ts_manager.cc:194] Registered new tserver with Master: eeb9132d8364461bb4446d584eb2da3e (127.10.152.3:40657)
14:14:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:54.942791 10848 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:14:54.956 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:14:54.956 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:14:54.957 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.10.152.62"
port: 41699
}
bound_http_address {
host: "127.10.152.62"
port: 34997
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.10.152.61"
port: 44711
}
bound_http_address {
host: "127.10.152.61"
port: 44945
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.10.152.60"
port: 39017
}
bound_http_address {
host: "127.10.152.60"
port: 44497
}
}
}
14:14:54.957 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:14:54.958 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.10.152.1"
port: 35767
}
bound_http_address {
host: "127.10.152.1"
port: 36325
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.10.152.2"
port: 46737
}
bound_http_address {
host: "127.10.152.2"
port: 45739
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.10.152.3"
port: 40657
}
bound_http_address {
host: "127.10.152.3"
port: 34447
}
}
}
14:14:54.959 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:14:54.960 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster17763127877166933940/krb5kdc/krb5cc
14:14:54.960 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:14:54.961 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLlQHbR/test-tmp
14:14:54.962 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:14:54.962 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLlQHbR/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:14:54.965 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:14:55.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:14:55.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:14:55.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:55.306545 11570 mini_oidc.cc:150] Starting JWKS server
14:14:55.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:55.309940 11570 webserver.cc:492] Webserver started at http://127.0.0.1:34487/ using document root <none> and password file <none>
14:14:55.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:55.310623 11570 mini_oidc.cc:186] Starting OIDC Discovery server
14:14:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:55.313072 11570 webserver.cc:492] Webserver started at http://127.0.0.1:36493/ using document root <none> and password file <none>
14:14:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:14:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/principal' for realm 'KRBTEST.COM',
14:14:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:14:55.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:14:55 dist-test-slave-vb5z krb5kdc[11581](info): setting up network...
14:14:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:14:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:14:55 dist-test-slave-vb5z krb5kdc[11581](info): set up 2 sockets
14:14:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:14:55 dist-test-slave-vb5z krb5kdc[11581](info): commencing operation
14:14:55.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:14:55.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:55.934913 11559 heartbeater.cc:499] Master 127.10.152.60:39017 was elected leader, sending a full tablet report...
14:14:57.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:57.475045 11570 mini_kdc.cc:121] Time spent starting KDC: real 2.123s user 0.008s sys 0.013s
14:14:57.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:14:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:14:57.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:14:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:14:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:14:57.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:57.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:14:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:14:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/test-user.keytab.
14:14:57.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/test-user.keytab.
14:14:57.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:14:57 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:57.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:14:57.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:14:57Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:14:57.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:14:57Z Disabled control of system clock
14:14:57.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.76.190@KRBTEST.COM; defaulting to no policy
14:14:57.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:57.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.76.190@KRBTEST.COM" created.
14:14:57.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:57.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.76.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:57.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.76.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:57.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.76.190@KRBTEST.COM; defaulting to no policy
14:14:57.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:57.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.76.190@KRBTEST.COM" created.
14:14:57.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:57.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.76.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:57.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.76.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:57.715665 11570 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/logs
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/info.pb
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.190:46649
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.190
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.190
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:57.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.040848 11598 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:14:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.041400 11598 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:14:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.041949 11598 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:14:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.075770 11598 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:14:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.076011 11598 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:14:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.076292 11598 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:14:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.076493 11598 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:14:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.076690 11598 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:14:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.076885 11598 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:14:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.077059 11598 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34487/jwks.json
14:14:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:58.117352 11598 master_runner.cc:386] Master server non-default flags:
14:14:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:14:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:58.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.190
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:58.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.190:46649
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/info.pb
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.190
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/logs
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:14:58.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:58.119012 11598 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:14:58.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:58.120396 11598 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:14:58.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.134168 11605 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:59.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.141742 11607 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:59.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:58.135299 11604 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:14:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:59.498505 11606 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1361 milliseconds
14:14:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.498626 11598 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:14:59.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.499929 11598 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:14:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.502372 11598 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:14:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.503784 11598 hybrid_clock.cc:648] HybridClock initialized: now 1777904099503713 us; error 53 us; skew 500 ppm
14:14:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.504756 11598 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:14:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:14:59 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904099, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:14:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.510598 11598 init.cc:377] Logged in from keytab as kudu/127.11.76.190@KRBTEST.COM (short username kudu)
14:14:59.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.516145 11598 webserver.cc:492] Webserver started at http://127.11.76.190:45289/ using document root <none> and password file <none>
14:14:59.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.517089 11598 fs_manager.cc:362] Metadata directory not provided
14:14:59.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.517338 11598 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:14:59.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.517835 11598 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:14:59.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.522269 11598 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/instance:
14:14:59.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed597ffaf23045369e3873038f0eb6fe"
14:14:59.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:59 on dist-test-slave-vb5z"
14:14:59.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.523627 11598 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal/instance:
14:14:59.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed597ffaf23045369e3873038f0eb6fe"
14:14:59.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:59 on dist-test-slave-vb5z"
14:14:59.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.531008 11598 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
14:14:59.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.536655 11616 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:14:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.537767 11598 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:14:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.538163 11598 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:14:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed597ffaf23045369e3873038f0eb6fe"
14:14:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:59 on dist-test-slave-vb5z"
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.538501 11598 fs_report.cc:389] FS layout report
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/data
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:14:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:14:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.599603 11598 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:14:59.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.605628 11598 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:14:59.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.606276 11598 kserver.cc:163] Server-wide thread pool size limit: 3276
14:14:59.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.684079 11598 rpc_server.cc:307] RPC server started. Bound to: 127.11.76.190:46649
14:14:59.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.684152 11668 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.76.190:46649 every 8 connection(s)
14:14:59.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.686903 11598 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/info.pb
14:14:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.692935 11669 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:14:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.695677 11570 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11598
14:14:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.696161 11570 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal/instance
14:14:59.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.704025 11669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } has no permanent_uuid. Determining permanent_uuid...
14:14:59.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.76.189@KRBTEST.COM; defaulting to no policy
14:14:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.76.189@KRBTEST.COM" created.
14:14:59.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:14:59 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904099, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.190@KRBTEST.COM for kudu/127.11.76.190@KRBTEST.COM
14:14:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.76.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:59.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.76.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:59.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.746399 11669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } has no permanent_uuid. Determining permanent_uuid...
14:14:59.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:59.749889 11620 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:14:59.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:59.755635 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.189:35707: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:14:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.76.189@KRBTEST.COM; defaulting to no policy
14:14:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:59.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.76.189@KRBTEST.COM" created.
14:14:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:14:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.76.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:59.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.76.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:14:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.792532 11570 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:14:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:14:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data
14:14:59.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/logs
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/info.pb
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.189:35707
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.189
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.189
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:14:59.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:14:59.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.805241 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } attempt: 1
14:14:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:59.810055 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.189:35707: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:14:59.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.878520 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } attempt: 2
14:14:59.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:59.882860 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.189:35707: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:14:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:14:59.974247 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } attempt: 3
14:14:59.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:14:59.978366 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.189:35707: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.112372 11678 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:15:00.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.113061 11678 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:15:00.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.113725 11678 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:15:00.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:00.121809 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } attempt: 4
14:15:00.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.126345 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.189:35707: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.149942 11678 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:15:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.150179 11678 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:15:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.150449 11678 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:15:00.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.150648 11678 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:15:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.150977 11678 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:15:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.151301 11678 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:15:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.151584 11678 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34487/jwks.json
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:00.193126 11678 master_runner.cc:386] Master server non-default flags:
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.189
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.189:35707
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:00.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/info.pb
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.189
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/logs
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:15:00.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:15:00.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:00.194777 11678 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:15:00.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:00.196157 11678 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:15:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.210196 11684 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:00.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:00.425833 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } attempt: 5
14:15:00.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.433423 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.189:35707: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:00.981058 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } attempt: 6
14:15:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.009721 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.189:35707: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:01.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.615192 11683 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 11678
14:15:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.662328 11683 kernel_stack_watchdog.cc:198] Thread 11678 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
14:15:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:15:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:15:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:15:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:15:01.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:00.210388 11685 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:01.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.670514 11686 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1453 milliseconds
14:15:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.670599 11678 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.461s user 0.004s sys 0.003s
14:15:01.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.671042 11678 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.462s user 0.004s sys 0.003s
14:15:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.672761 11688 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.672863 11678 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:15:01.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.674223 11678 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:01.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.676838 11678 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:01.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.678283 11678 hybrid_clock.cc:648] HybridClock initialized: now 1777904101678247 us; error 38 us; skew 500 ppm
14:15:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.679178 11678 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:01.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:01 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904101, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:01.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.687393 11678 init.cc:377] Logged in from keytab as kudu/127.11.76.189@KRBTEST.COM (short username kudu)
14:15:01.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.700364 11678 webserver.cc:492] Webserver started at http://127.11.76.189:33047/ using document root <none> and password file <none>
14:15:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.701262 11678 fs_manager.cc:362] Metadata directory not provided
14:15:01.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.701457 11678 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:01.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.701895 11678 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:15:01.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.706291 11678 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/instance:
14:15:01.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee0e177bc11a419f80caef961ccf2521"
14:15:01.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:01 on dist-test-slave-vb5z"
14:15:01.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.707423 11678 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal/instance:
14:15:01.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee0e177bc11a419f80caef961ccf2521"
14:15:01.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:01 on dist-test-slave-vb5z"
14:15:01.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.716059 11678 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
14:15:01.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.722139 11699 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:01.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.723433 11678 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.001s
14:15:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.723882 11678 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee0e177bc11a419f80caef961ccf2521"
14:15:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:01 on dist-test-slave-vb5z"
14:15:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.724212 11678 fs_report.cc:389] FS layout report
14:15:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:01.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/data
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:01.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:01.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.797442 11678 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.807768 11678 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:01.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.808226 11678 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:01.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.886139 11678 rpc_server.cc:307] RPC server started. Bound to: 127.11.76.189:35707
14:15:01.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.886214 11750 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.76.189:35707 every 8 connection(s)
14:15:01.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.889223 11678 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/info.pb
14:15:01.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.892534 11570 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11678
14:15:01.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.893065 11570 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal/instance
14:15:01.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.897008 11751 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:15:01.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.76.188@KRBTEST.COM; defaulting to no policy
14:15:01.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.909842 11751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } has no permanent_uuid. Determining permanent_uuid...
14:15:01.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:15:01.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.76.188@KRBTEST.COM" created.
14:15:01.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:01 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904101, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.189@KRBTEST.COM for kudu/127.11.76.190@KRBTEST.COM
14:15:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:15:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.76.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:15:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.76.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:15:01.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.952356 11751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } has no permanent_uuid. Determining permanent_uuid...
14:15:01.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.11.76.188@KRBTEST.COM; defaulting to no policy
14:15:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:15:01.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.11.76.188@KRBTEST.COM" created.
14:15:01.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:01 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904101, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.189@KRBTEST.COM for kudu/127.11.76.189@KRBTEST.COM
14:15:01.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.981397 11751 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } has no permanent_uuid. Determining permanent_uuid...
14:15:01.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.984098 11700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:01.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:01.988523 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:15:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.76.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:15:01.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.11.76.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab.
14:15:01.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:01.992604 11570 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/logs
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/info.pb
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.188:34283
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.188
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.188
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:15:01.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:15:02.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.038098 11751 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 1
14:15:02.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.042878 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.070418 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } attempt: 7
14:15:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:02 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904099, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.190@KRBTEST.COM for kudu/127.11.76.189@KRBTEST.COM
14:15:02.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.096067 11669 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } has no permanent_uuid. Determining permanent_uuid...
14:15:02.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.102092 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.111450 11751 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 2
14:15:02.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.116479 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.160632 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 1
14:15:02.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.165764 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.207958 11751 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 3
14:15:02.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.212842 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.247234 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 2
14:15:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.251255 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.331429 11761 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:15:02.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.332055 11761 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:15:02.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.332615 11761 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:15:02.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.336745 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 3
14:15:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.342682 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.356328 11751 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 4
14:15:02.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.360488 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.370949 11761 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:15:02.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.371183 11761 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:15:02.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.371605 11761 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:15:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.371958 11761 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:15:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.372280 11761 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:15:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.372598 11761 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:15:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.372848 11761 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34487/jwks.json
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.420007 11761 master_runner.cc:386] Master server non-default flags:
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.188
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.188:34283
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/info.pb
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.188
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/logs
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:02.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:15:02.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.421303 11761 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:15:02.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.422322 11761 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:15:02.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.433293 11769 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:02.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.483227 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 4
14:15:02.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.491909 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.659963 11751 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 5
14:15:02.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.670470 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:02.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:02.775565 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 5
14:15:02.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.783258 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:03.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:03.218063 11751 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 6
14:15:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:03.240674 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:03.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:03.335793 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 6
14:15:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:03.362121 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:04.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:02.434921 11770 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:03.838863 11768 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 11761
14:15:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.204974 11768 kernel_stack_watchdog.cc:198] Thread 11761 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
14:15:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:15:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:15:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:15:04.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:15:04.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.210320 11761 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.776s user 0.005s sys 0.001s
14:15:04.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.210784 11761 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.777s user 0.005s sys 0.001s
14:15:04.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.211594 11771 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1769 milliseconds
14:15:04.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.212587 11772 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:04.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.212664 11761 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:15:04.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.214056 11761 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:04.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.217109 11761 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:04.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.218555 11761 hybrid_clock.cc:648] HybridClock initialized: now 1777904104218487 us; error 75 us; skew 500 ppm
14:15:04.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.219352 11761 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:04.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:04.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.226135 11761 init.cc:377] Logged in from keytab as kudu/127.11.76.188@KRBTEST.COM (short username kudu)
14:15:04.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.232100 11761 webserver.cc:492] Webserver started at http://127.11.76.188:41365/ using document root <none> and password file <none>
14:15:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.233064 11761 fs_manager.cc:362] Metadata directory not provided
14:15:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.233278 11761 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:04.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.233696 11761 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
14:15:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.238531 11761 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/instance:
14:15:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b22a93d405441e8b833a2ab7ae9ef8f"
14:15:04.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:04 on dist-test-slave-vb5z"
14:15:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.239818 11761 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal/instance:
14:15:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b22a93d405441e8b833a2ab7ae9ef8f"
14:15:04.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:04 on dist-test-slave-vb5z"
14:15:04.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.248013 11761 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.004s
14:15:04.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.253702 11786 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:04.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.254860 11761 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
14:15:04.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.255293 11761 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:04.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b22a93d405441e8b833a2ab7ae9ef8f"
14:15:04.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:04 on dist-test-slave-vb5z"
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.255764 11761 fs_report.cc:389] FS layout report
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/data
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:04.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:04.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.301162 11751 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 7
14:15:04.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.309012 11751 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:04.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.321375 11761 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.329332 11761 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.329936 11761 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.395750 11669 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } attempt: 7
14:15:04.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.404130 11669 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.76.188:34283: Network error: Client connection negotiation failed: client connection to 127.11.76.188:34283: connect: Connection refused (error 111)
14:15:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.414839 11761 rpc_server.cc:307] RPC server started. Bound to: 127.11.76.188:34283
14:15:04.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.414916 11842 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.76.188:34283 every 8 connection(s)
14:15:04.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.417686 11761 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/info.pb
14:15:04.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.419813 11570 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11761
14:15:04.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.420378 11570 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal/instance
14:15:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.424515 11843 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:15:04.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.438110 11843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } has no permanent_uuid. Determining permanent_uuid...
14:15:04.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.76.188@KRBTEST.COM
14:15:04.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for kudu/127.11.76.190@KRBTEST.COM
14:15:04.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.471237 11843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } has no permanent_uuid. Determining permanent_uuid...
14:15:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.76.189@KRBTEST.COM
14:15:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for kudu/127.11.76.189@KRBTEST.COM
14:15:04.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.498301 11843 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } has no permanent_uuid. Determining permanent_uuid...
14:15:04.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904097, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.76.190@KRBTEST.COM
14:15:04.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for kudu/127.11.76.188@KRBTEST.COM
14:15:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.531193 11570 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
14:15:04.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.531330 11843 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Bootstrap starting.
14:15:04.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.537057 11843 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Neither blocks nor log segments found. Creating new log.
14:15:04.538 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:04.538 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:15:04.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.539100 11843 log.cc:826] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Log is configured to *not* fsync() on all Append() calls
14:15:04.540 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:15:04.540 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:15:04.541 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.76.190"
port: 46649
}
bound_http_address {
host: "127.11.76.190"
port: 45289
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.76.189"
port: 35707
}
bound_http_address {
host: "127.11.76.189"
port: 33047
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.76.188"
port: 34283
}
bound_http_address {
host: "127.11.76.188"
port: 41365
}
}
}
14:15:04.541 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:15:04.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:04.542 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:15:04.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:15:04.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.544085 11843 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: No bootstrap required, opened a new log
14:15:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.564428 11843 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:04.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.565469 11843 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:15:04.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.565840 11843 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5b22a93d405441e8b833a2ab7ae9ef8f, State: Initialized, Role: FOLLOWER
14:15:04.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.566891 11843 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:04.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.570749 11854 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } } }
14:15:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.571764 11854 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [sys.catalog]: This master's current role is: FOLLOWER
14:15:04.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.572153 11843 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [sys.catalog]: configured and running, proceeding with master startup.
14:15:04.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:04 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:04.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:15:04.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:04.582 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc
14:15:04.584 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:15:04.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.586470 11795 tablet.cc:2404] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:15:04.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.600514 11865 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:04.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:04.600834 11865 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.597295 11761 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: OK, instance_id { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" instance_seqno: 1777904099662338 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:15:04.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.627715 11761 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:04.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.637439 11854 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:04.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.638029 11854 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:04.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:04.640269 11854 leader_election.cc:290] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649), ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707)
14:15:05.585 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.76.190:46649
14:15:05.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:15:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:05.586169 11570 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11598
14:15:05.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:05.606031 11865 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:05.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:05.606427 11865 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:05.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:05.619225 11788 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.11.76.190:46649 (error 108)
14:15:05.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:05.629 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.76.188:34283
14:15:05.630 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:15:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:05.630627 11570 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11761
14:15:05.650 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:05.650 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.76.189:35707
14:15:05.650 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:15:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:05.650713 11570 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11678
14:15:05.671 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:05.672 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.76.190:46649
14:15:05.672 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:05.674038 11570 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/logs
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/info.pb
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:15:05.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.190:46649
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.190
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45289
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.190
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:15:05.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:15:06.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.013453 11869 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:15:06.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.014137 11869 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:15:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.014874 11869 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:15:06.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.048296 11869 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:15:06.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.048617 11869 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:15:06.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.049062 11869 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:15:06.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.049371 11869 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:15:06.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.049674 11869 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:15:06.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.049976 11869 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:15:06.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.050223 11869 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34487/jwks.json
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:06.090706 11869 master_runner.cc:386] Master server non-default flags:
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.190
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.190:46649
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/info.pb
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.190
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45289
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:15:06.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:15:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:15:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/logs
14:15:06.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:06.092399 11869 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:15:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:06.093667 11869 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:15:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.107667 11875 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.109684 11878 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:06.107885 11876 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:07.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.407063 11877 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1296 milliseconds
14:15:07.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.407191 11869 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:15:07.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.408555 11869 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.411775 11869 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:07.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.413234 11869 hybrid_clock.cc:648] HybridClock initialized: now 1777904107413199 us; error 67 us; skew 500 ppm
14:15:07.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.414270 11869 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:07.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:07 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904107, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:07.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.420645 11869 init.cc:377] Logged in from keytab as kudu/127.11.76.190@KRBTEST.COM (short username kudu)
14:15:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.425428 11869 webserver.cc:492] Webserver started at http://127.11.76.190:45289/ using document root <none> and password file <none>
14:15:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.426407 11869 fs_manager.cc:362] Metadata directory not provided
14:15:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.426617 11869 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.435154 11869 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
14:15:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.439841 11887 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.441041 11869 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:15:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.441448 11869 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:15:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ed597ffaf23045369e3873038f0eb6fe"
14:15:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:14:59 on dist-test-slave-vb5z"
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.443480 11869 fs_report.cc:389] FS layout report
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/data
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:07.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.511250 11869 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:07.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.522562 11869 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:07.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.523130 11869 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.600131 11869 rpc_server.cc:307] RPC server started. Bound to: 127.11.76.190:46649
14:15:07.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.600219 11938 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.76.190:46649 every 8 connection(s)
14:15:07.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.602856 11869 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/data/info.pb
14:15:07.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.611783 11570 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11869
14:15:07.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:07.612 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.76.188:34283
14:15:07.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.613993 11570 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/logs
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/info.pb
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.188:34283
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.188
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41365
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.188
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:15:07.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.615043 11939 sys_catalog.cc:263] Verifying existing consensus state
14:15:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260504 14:15:07.615921 11939 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:15:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.616328 11869 master.cc:562] Master@127.11.76.190:46649 shutting down...
14:15:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:07.677735 11869 master.cc:584] Master@127.11.76.190:46649 shutdown complete.
14:15:07.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.680433 11869 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
14:15:07.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:15:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.947703 11941 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:15:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.948271 11941 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:15:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.948868 11941 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:15:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.982911 11941 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:15:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.983153 11941 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:15:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.983418 11941 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:15:07.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.983705 11941 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:15:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.983908 11941 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:15:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.984112 11941 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:15:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:07.984407 11941 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34487/jwks.json
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:08.027868 11941 master_runner.cc:386] Master server non-default flags:
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.188
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.188:34283
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/info.pb
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.188
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41365
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:15:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/logs
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:15:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:08.029109 11941 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:15:08.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:08.030234 11941 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:15:08.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:08.040948 11947 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:09.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.444903 11946 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 11941
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.478605 11946 kernel_stack_watchdog.cc:198] Thread 11941 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.478994 11941 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.438s user 0.465s sys 0.947s
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:08.041857 11948 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:09.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.479414 11941 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.438s user 0.465s sys 0.947s
14:15:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.482172 11950 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:09.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.486756 11949 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1444 milliseconds
14:15:09.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.486759 11941 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:15:09.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.488276 11941 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:09.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.490830 11941 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:09.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.492221 11941 hybrid_clock.cc:648] HybridClock initialized: now 1777904109492175 us; error 78 us; skew 500 ppm
14:15:09.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.493038 11941 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:09.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:09 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904109, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.499781 11941 init.cc:377] Logged in from keytab as kudu/127.11.76.188@KRBTEST.COM (short username kudu)
14:15:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.505455 11941 webserver.cc:492] Webserver started at http://127.11.76.188:41365/ using document root <none> and password file <none>
14:15:09.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.506632 11941 fs_manager.cc:362] Metadata directory not provided
14:15:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.506924 11941 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:09.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.516749 11941 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.004s
14:15:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.521461 11961 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:09.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.522536 11941 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.522938 11941 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5b22a93d405441e8b833a2ab7ae9ef8f"
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:04 on dist-test-slave-vb5z"
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.524996 11941 fs_report.cc:389] FS layout report
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/wal
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/data
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:09.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:09.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:09.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.581015 11941 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:09.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.587314 11941 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:09.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.588126 11941 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.664770 11941 rpc_server.cc:307] RPC server started. Bound to: 127.11.76.188:34283
14:15:09.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.664851 12013 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.76.188:34283 every 8 connection(s)
14:15:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.667778 11941 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-2/data/info.pb
14:15:09.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.674587 11570 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 11941
14:15:09.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:09.675 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.76.189:35707
14:15:09.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.676811 11570 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/logs
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/info.pb
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.189:35707
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.189
14:15:09.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33047
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.189
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:15:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.678519 12014 sys_catalog.cc:263] Verifying existing consensus state
14:15:09.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.686601 12014 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Bootstrap starting.
14:15:09.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.737794 12014 log.cc:826] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Log is configured to *not* fsync() on all Append() calls
14:15:09.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.743914 12014 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:15:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.744861 12014 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Bootstrap complete.
14:15:09.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.775038 12014 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:09.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.777431 12014 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:15:09.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.777772 12014 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5b22a93d405441e8b833a2ab7ae9ef8f, State: Initialized, Role: FOLLOWER
14:15:09.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.778723 12014 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.781452 12018 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } } }
14:15:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.782301 12018 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [sys.catalog]: This master's current role is: FOLLOWER
14:15:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.782816 12014 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [sys.catalog]: configured and running, proceeding with master startup.
14:15:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.814622 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.815012 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.809989 11941 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111),
14:15:09.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:09.839265 11941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:09.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:09.843902 11968 tablet.cc:2404] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:15:10.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.084080 12016 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
14:15:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.084805 12016 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:15:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.085726 12016 flags.cc:432] Enabled unsafe flag: --never_fsync=true
14:15:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.129269 12016 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
14:15:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.129711 12016 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
14:15:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.130168 12016 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
14:15:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.130499 12016 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
14:15:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.130841 12016 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
14:15:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.131193 12016 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
14:15:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.131444 12016 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:34487/jwks.json
14:15:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.144773 12018 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:10.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.145368 12018 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.150926 11963 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111)
14:15:10.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.153210 12018 leader_election.cc:290] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649), ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707)
14:15:10.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.169373 11963 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649): Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111)
14:15:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.173343 11964 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707): Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.173864 11964 leader_election.cc:304] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b22a93d405441e8b833a2ab7ae9ef8f; no voters: ed597ffaf23045369e3873038f0eb6fe, ee0e177bc11a419f80caef961ccf2521
14:15:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.175850 12018 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.204783 12016 master_runner.cc:386] Master server non-default flags:
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.76.148:42795
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.76.190:46649,127.11.76.189:35707,127.11.76.188:34283
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/kudu.keytab
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.76.189
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.76.189:35707
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:15:10.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34487/jwks.json
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/info.pb
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.76.189
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33047
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/logs
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 0ba7f54e15c9f673bf3e350f30b8d5f1a4037b79
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 May 2026 13:43:23 UTC on e7f111948823
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11743
14:15:10.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:15:10.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.208489 12016 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:15:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:10.210369 12016 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:15:10.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.241791 12038 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:10.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.828666 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:10.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.830952 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:11.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:10.243649 12039 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:11.673259 12016 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.432s user 0.534s sys 0.810s
14:15:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:11.645237 12037 debug-util.cc:398] Leaking SignalData structure 0x7b0800028d00 after lost signal to thread 12016
14:15:11.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:11.674765 12016 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.434s user 0.534s sys 0.810s
14:15:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.692209 12016 server_base.cc:1061] running on GCE node
14:15:11.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:11.694149 12045 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:15:11.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.696590 12016 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:15:11.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.699705 12016 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:15:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.701181 12016 hybrid_clock.cc:648] HybridClock initialized: now 1777904111701101 us; error 79 us; skew 500 ppm
14:15:11.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.702293 12016 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:15:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:11 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904111, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.710460 12016 init.cc:377] Logged in from keytab as kudu/127.11.76.189@KRBTEST.COM (short username kudu)
14:15:11.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.717020 12016 webserver.cc:492] Webserver started at http://127.11.76.189:33047/ using document root <none> and password file <none>
14:15:11.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.718310 12016 fs_manager.cc:362] Metadata directory not provided
14:15:11.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.718626 12016 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:15:11.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.729527 12016 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.008s sys 0.000s
14:15:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.735221 12051 log_block_manager.cc:3869] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:15:11.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.736591 12016 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
14:15:11.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.737015 12016 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data,/tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ee0e177bc11a419f80caef961ccf2521"
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-05-04 14:15:01 on dist-test-slave-vb5z"
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.739886 12016 fs_report.cc:389] FS layout report
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/data
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:15:11.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:15:11.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.823596 12016 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:15:11.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.831192 12016 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:15:11.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.831815 12016 kserver.cc:163] Server-wide thread pool size limit: 3276
14:15:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:11.834805 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:11.835301 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:11.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.924002 12016 rpc_server.cc:307] RPC server started. Bound to: 127.11.76.189:35707
14:15:11.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.924041 12102 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.76.189:35707 every 8 connection(s)
14:15:11.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.927969 12016 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/data/info.pb
14:15:11.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.928563 11570 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu as pid 12016
14:15:11.929 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:15:11.930 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:11.932 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.76.188:34283(127.11.76.188:34283)] connecting to peer
14:15:11.935 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:11.938 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:11.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:11.940 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:11.941 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.11.76.188:34283(127.11.76.188:34283)] Successfully connected to peer
14:15:11.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.950244 12103 sys_catalog.cc:263] Verifying existing consensus state
14:15:11.952 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.11.76.189:35707(127.11.76.189:35707)] Successfully connected to peer
14:15:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260504 14:15:11.956256 12103 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:15:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:11.958035 12016 master.cc:562] Master@127.11.76.189:35707 shutting down...
14:15:11.979 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:15:11.980 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc
14:15:11.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:11 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:11.993 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:15:11.993 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
14:15:11.993 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
14:15:11.994 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/krb5kdc/krb5cc
14:15:11.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:11 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:11.999 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:15:11.999 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
14:15:12.000 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.11.76.189
14:15:12.003 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.11.76.188
14:15:12.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:12 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.76.189@KRBTEST.COM
14:15:12.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:12 dist-test-slave-vb5z krb5kdc[11581](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1777904104, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.76.188@KRBTEST.COM
14:15:12.048 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xcecc01d2, L:/127.0.0.1:43470 - R:/127.11.76.189:35707] using SASL/GSSAPI
14:15:12.054 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.11.76.189:35707(127.11.76.189:35707)] handling channelInactive
14:15:12.054 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state READY due to: connection closed
14:15:12.055 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: connection closed
14:15:12.064 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x430a37be, L:/127.0.0.1:57164 - R:/127.11.76.188:34283] using SASL/GSSAPI
14:15:12.069 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.091 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.093 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.097 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.097 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.098 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.101 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.102 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.102 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.102 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.111 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.113 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.113 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.113 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.113 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.116 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.116 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.116 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.118 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:12.120337 12016 master.cc:584] Master@127.11.76.189:35707 shutdown complete.
14:15:12.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:12.123857 12016 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
14:15:12.130 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskLlQHbR/test-tmp/mini-kudu-cluster18408952609853982720/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
14:15:12.136 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.137 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.137 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.137 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.141 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.141 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.141 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.141 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.151 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.153 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.153 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.153 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.154 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.155 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.155 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.155 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.158 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.171 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.172 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.172 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.172 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.172 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.176 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.177 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.177 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.177 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.231 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.232 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.232 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.232 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.232 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.233 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.233 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.233 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.235 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:12.249600 12116 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:12.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:12.250149 12116 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:12.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:12.252277 12116 leader_election.cc:290] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649), ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707)
14:15:12.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:12.259835 11964 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707): Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:12.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:12.260537 11963 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649): Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111)
14:15:12.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:12.260892 11963 leader_election.cc:304] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b22a93d405441e8b833a2ab7ae9ef8f; no voters: ed597ffaf23045369e3873038f0eb6fe, ee0e177bc11a419f80caef961ccf2521
14:15:12.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:12.261663 12116 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:15:12.310 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.311 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.312 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.312 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.312 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.313 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.313 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.313 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.315 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.491 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.492 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.493 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.493 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.493 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.493 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.493 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.496 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.711 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:12.712 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.712 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.712 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:12.713 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:12.714 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.714 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.714 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:12.715 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:12.838346 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:12.838688 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:13.371 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:13.372 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:13.373 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:13.373 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:13.373 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:13.374 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:13.374 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:13.374 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:13.377 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:13.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:13.841583 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:13.841907 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:14.038951 12123 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:14.039367 12123 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:14.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:14.040858 12123 leader_election.cc:290] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649), ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707)
14:15:14.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:14.047433 11963 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649): Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111)
14:15:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:14.047904 11964 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707): Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:14.048254 11964 leader_election.cc:304] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b22a93d405441e8b833a2ab7ae9ef8f; no voters: ed597ffaf23045369e3873038f0eb6fe, ee0e177bc11a419f80caef961ccf2521
14:15:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:14.048769 12123 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:15:14.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:14.842319 11941 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111),
14:15:14.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:14.842801 11941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:14.844940 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:14.845187 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:15.271 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:15.272 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:15.272 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:15.272 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:15.272 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:15.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:15.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:15.273 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:15.275 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:15.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:15.848057 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:15.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:15.848325 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:16.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:16.851114 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:16.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:16.851392 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:17.301632 12131 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:17.302146 12131 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:17.304122 12131 leader_election.cc:290] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649), ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707)
14:15:17.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:17.306543 11964 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111) [suppressed 5 similar messages]
14:15:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:17.309711 11964 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707): Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:17.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:17.310278 11963 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649): Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111)
14:15:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:17.310616 11963 leader_election.cc:304] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b22a93d405441e8b833a2ab7ae9ef8f; no voters: ed597ffaf23045369e3873038f0eb6fe, ee0e177bc11a419f80caef961ccf2521
14:15:17.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:17.311208 12131 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:15:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:17.854382 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:17.854722 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:18.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:18.858215 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:18.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:18.858655 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:18.891 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:18.892 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:18.893 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:18.893 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:18.893 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:18.893 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:18.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:18.893 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:18.895 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:19.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:19 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904119, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:19.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:19.503875 11957 init.cc:295] Successfully reacquired a new kerberos TGT
14:15:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:19.666437 11275 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad060 after lost signal to thread 11148
14:15:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:19.667654 11275 debug-util.cc:398] Leaking SignalData structure 0x7b08000b64a0 after lost signal to thread 11278
14:15:19.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:19.846814 11941 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111),
14:15:19.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:19.847209 11941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:19.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:19.861833 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:19.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:19.862100 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:20.291 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:20.292 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:20.292 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:20.292 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:20.292 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:20.294 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:20.294 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:20.294 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:20.296 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:20.864969 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:20.865231 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:21.868232 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:21.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:21.868497 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:22.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:22.872190 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:22.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:22.872545 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:22.951 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:22.952 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:22.952 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:22.953 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:22.953 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:22.954 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:22.954 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:22.954 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:22.957 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:23.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:23.876544 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:23.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:23.876906 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:24.112341 12145 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:24.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:24.113070 12145 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:24.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:24.114550 12145 leader_election.cc:290] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649), ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707)
14:15:24.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:24.116842 11963 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:15:24.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:24.119863 11964 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707): Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:24.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:24.120347 11963 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649): Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111)
14:15:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:24.120687 11963 leader_election.cc:304] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b22a93d405441e8b833a2ab7ae9ef8f; no voters: ed597ffaf23045369e3873038f0eb6fe, ee0e177bc11a419f80caef961ccf2521
14:15:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:24.121224 12145 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:15:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:24.850069 11941 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111),
14:15:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:24.850466 11941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:24.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:24.880863 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:24.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:24.881148 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:25.883950 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:25.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:25.884294 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:26.151 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:26.152 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:26.152 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:26.152 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:26.153 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:26.154 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:26.154 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:26.154 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:26.155 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:26.887496 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:26.887755 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:27.890820 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:27.891148 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:28.231 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:28.232 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:28.233 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:28.233 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:28.233 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:28.233 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:28.233 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:28.233 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:28.236 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:28.894364 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:28.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:28.894726 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:29.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:29 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904129, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:29.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:29.506673 11957 init.cc:295] Successfully reacquired a new kerberos TGT
14:15:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:29.854039 11941 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111),
14:15:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:29.854486 11941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:29.897680 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:29.897979 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:30.450 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:30.451 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:30.452 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:30.452 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:30.452 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:30.452 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:30.452 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:30.453 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:30.455 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:30.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:30.900707 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:30.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:30.900978 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:31.903906 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:31.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:31.904184 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:32.115636 12158 raft_consensus.cc:493] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:15:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:32.116113 12158 raft_consensus.cc:515] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ed597ffaf23045369e3873038f0eb6fe" member_type: VOTER last_known_addr { host: "127.11.76.190" port: 46649 } } peers { permanent_uuid: "ee0e177bc11a419f80caef961ccf2521" member_type: VOTER last_known_addr { host: "127.11.76.189" port: 35707 } } peers { permanent_uuid: "5b22a93d405441e8b833a2ab7ae9ef8f" member_type: VOTER last_known_addr { host: "127.11.76.188" port: 34283 } }
14:15:32.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:32.117766 12158 leader_election.cc:290] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649), ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707)
14:15:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:32.120522 11963 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111) [suppressed 1 similar messages]
14:15:32.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:32.123694 11963 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ed597ffaf23045369e3873038f0eb6fe (127.11.76.190:46649): Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111)
14:15:32.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:32.124308 11964 leader_election.cc:336] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer ee0e177bc11a419f80caef961ccf2521 (127.11.76.189:35707): Network error: Client connection negotiation failed: client connection to 127.11.76.189:35707: connect: Connection refused (error 111)
14:15:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:32.124713 11964 leader_election.cc:304] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 5b22a93d405441e8b833a2ab7ae9ef8f; no voters: ed597ffaf23045369e3873038f0eb6fe, ee0e177bc11a419f80caef961ccf2521
14:15:32.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:32.125416 12158 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
14:15:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:32.906852 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:32.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:32.907141 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:33.411 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:33.413 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:33.413 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:33.413 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:33.413 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:33.414 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:33.414 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:33.414 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:33.416 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:33.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:33.910070 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:33.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:33.910380 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:34.251 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:34.252 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:34.252 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:34.252 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:34.252 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:34.253 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:34.253 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:34.253 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:34.255 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:34.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:34.546514 12010 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad9e0 after lost signal to thread 11942
14:15:34.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:34.547175 12010 debug-util.cc:398] Leaking SignalData structure 0x7b08000aeba0 after lost signal to thread 12013
14:15:34.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:34.903442 11941 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111),
14:15:34.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:34.903940 11941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:34.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:34.914816 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:34.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:34.915195 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:35.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:35.919618 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:35.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:35.919998 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:36.051 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:36.051 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:36.052 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:36.052 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:36.052 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:36.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:36.053 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:36.066 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:36.067 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:36.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:36.922854 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:36.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:36.923117 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:37.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:37.738890 11275 debug-util.cc:398] Leaking SignalData structure 0x7b08000bace0 after lost signal to thread 11148
14:15:37.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:37.739851 11275 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad1e0 after lost signal to thread 11278
14:15:37.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:37.925971 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:37.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:37.926244 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:38.250 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:38.251 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:38.252 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:38.252 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:38.252 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:38.252 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:38.252 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:38.252 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:38.256 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:38.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:38.929673 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:38.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:38.930032 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:39.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 04 14:15:39 dist-test-slave-vb5z krb5kdc[11581](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1777904139, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.76.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:15:39.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:39.509518 11957 init.cc:295] Successfully reacquired a new kerberos TGT
14:15:39.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:39.907624 11941 master_runner.cc:186] Error getting master registration for 127.11.76.190:46649: Network error: Client connection negotiation failed: client connection to 127.11.76.190:46649: connect: Connection refused (error 111),
14:15:39.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:39.908074 11941 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:15:39.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:39.932601 12029 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 5b22a93d405441e8b833a2ab7ae9ef8f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:15:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260504 14:15:39.932862 12029 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:15:40.851 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.190:46649(127.11.76.190:46649)] connecting to peer
14:15:40.852 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.11.76.189:35707(127.11.76.189:35707)] connecting to peer
14:15:40.852 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:40.852 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.11.76.190:46649(127.11.76.190:46649)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:40.852 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.190:46649: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649
14:15:40.853 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:40.853 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.11.76.189:35707(127.11.76.189:35707)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:40.853 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.11.76.189:35707: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
14:15:40.854 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707; will retry
14:15:40.855 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [8ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [125ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: connection closed, [139ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [139ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: connection closed, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [167ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [168ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [172ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [186ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [188ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [207ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [210ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [211ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [211ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [225ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [227ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [242ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [245ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [247ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [247ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [302ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [303ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [305ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [305ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [380ms] refreshing cache from master, [380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [382ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [383ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [385ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [385ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [560ms] refreshing cache from master, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [566ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [780ms] refreshing cache from master, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [782ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [784ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1440ms] refreshing cache from master, [1441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [1443ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1447ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [1448ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3340ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [3342ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [3345ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [6960ms] refreshing cache from master, [6960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [6961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [6962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [6963ms] too many traces: truncated at 100 traces, deferred=Deferred@1894293402(state=PENDING, result=null, callback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307), errback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307)))
14:15:40.857 [DEBUG - main] (Connection.java:575) [peer master-127.11.76.189:35707(127.11.76.189:35707)] disconnecting while in state TERMINATED
14:15:40.858 [DEBUG - main] (Connection.java:575) [peer master-127.11.76.190:46649(127.11.76.190:46649)] disconnecting while in state TERMINATED
14:15:40.858 [DEBUG - main] (Connection.java:575) [peer master-127.11.76.188:34283(127.11.76.188:34283)] disconnecting while in state READY
14:15:40.858 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:15:40.860 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.11.76.188:34283(127.11.76.188:34283)] handling channelInactive
14:15:40.860 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.76.188:34283(127.11.76.188:34283)] cleaning up while in state READY due to: connection closed
14:15:40.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:40.861596 11570 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11941
14:15:42.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:15:42Z chronyd exiting
14:15:43.091 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:15:43.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:43.096088 10848 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11147
14:15:43.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:43.130136 10848 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11289
14:15:43.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:43.162494 10848 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11429
14:15:43.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:43.189666 10848 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 10860
14:15:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:43.225026 10848 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 10930
14:15:43.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260504 14:15:43.267969 10848 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLlQHbR/build/tsan/bin/kudu with pid 11005
14:15:44.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-05-04T14:15:44Z chronyd exiting
14:15:45.479 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [8ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [125ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: connection closed, [139ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [139ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: connection closed, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [167ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [168ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [172ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [186ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [188ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [207ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [210ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [211ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [211ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [225ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [227ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [242ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [245ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [247ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [247ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [302ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [303ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [305ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [305ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [380ms] refreshing cache from master, [380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [382ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [383ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [385ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [385ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [560ms] refreshing cache from master, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [566ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [780ms] refreshing cache from master, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [782ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [784ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1440ms] refreshing cache from master, [1441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [1443ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1447ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [1448ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3340ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [3342ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [3345ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [6960ms] refreshing cache from master, [6960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [6961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [6962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [6963ms] too many traces: truncated at 100 traces, deferred=Deferred@1894293402(state=PENDING, result=null, callback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307), errback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.76.189:35707
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [8ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [125ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: connection closed, [139ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [139ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: connection closed, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [167ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [168ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [172ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [186ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [188ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [207ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [210ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [211ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [211ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [225ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [227ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [242ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [245ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [247ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [247ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [302ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [303ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [305ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [305ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [380ms] refreshing cache from master, [380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [382ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [383ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [385ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [385ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [560ms] refreshing cache from master, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [566ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [780ms] refreshing cache from master, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [782ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [784ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1440ms] refreshing cache from master, [1441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [1443ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1447ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [1448ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3340ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [3342ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [3345ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [6960ms] refreshing cache from master, [6960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [6961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [6962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [6963ms] too many traces: truncated at 100 traces, deferred=Deferred@1894293402(state=PENDING, result=null, callback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307), errback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.76.189:35707
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
14:15:45.676 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
14:15:45.676 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [8ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [125ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: connection closed, [139ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [139ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: connection closed, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [167ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [168ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [172ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [186ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [188ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [207ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [210ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [211ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [211ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [225ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [227ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [242ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [245ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [247ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [247ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [302ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [303ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [305ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [305ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [380ms] refreshing cache from master, [380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [382ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [383ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [385ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [385ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [560ms] refreshing cache from master, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [566ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [780ms] refreshing cache from master, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [782ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [784ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1440ms] refreshing cache from master, [1441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [1443ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1447ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [1448ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3340ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [3342ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [3345ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [6960ms] refreshing cache from master, [6960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [6961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [6962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [6963ms] too many traces: truncated at 100 traces, deferred=Deferred@1894293402(state=PENDING, result=null, callback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307), errback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.76.189:35707
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=28925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [8ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [10ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [125ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: connection closed, [139ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [139ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: connection closed, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [163ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [167ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [168ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [172ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [172ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [183ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [186ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [188ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [189ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [205ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [206ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [207ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [210ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [211ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [211ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [223ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [225ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [227ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [242ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [245ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [247ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [247ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [300ms] refreshing cache from master, [301ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [302ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [303ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [305ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [305ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [380ms] refreshing cache from master, [380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [381ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [382ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [383ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [385ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [385ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [560ms] refreshing cache from master, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [562ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [563ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [566ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [566ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [780ms] refreshing cache from master, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [781ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [782ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [783ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [784ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [785ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [785ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1440ms] refreshing cache from master, [1441ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [1442ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [1443ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [1444ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [1447ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [1448ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3340ms] refreshing cache from master, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [3341ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [3342ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [3342ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.190:46649: Network error: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649, [3343ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.189:35707: Network error: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [3345ms] Sub RPC ConnectToMaster: received response from server master-127.11.76.188:34283: OK, [3345ms] delaying RPC due to: Service unavailable: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707, [6960ms] refreshing cache from master, [6960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.190:46649, [6961ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.188:34283, [6962ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.76.189:35707, [6963ms] too many traces: truncated at 100 traces, deferred=Deferred@1894293402(state=PENDING, result=null, callback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307), errback=(continuation of Deferred@127857422 after retry RPC after error@1076100223) -> (continuation of Deferred@85476553 after retry RPC after error@825761727) -> (continuation of Deferred@1104059413 after retry RPC after error@508641367) -> (continuation of Deferred@1504207192 after retry RPC after error@681330741) -> (continuation of Deferred@487016199 after retry RPC after error@155250440) -> (continuation of Deferred@335080921 after retry RPC after error@1500500595) -> (continuation of Deferred@2121767206 after retry RPC after error@1883308367) -> (continuation of Deferred@604224699 after retry RPC after error@64495106) -> (continuation of Deferred@1101273590 after retry RPC after error@966206452) -> (continuation of Deferred@13149721 after retry RPC after error@1134792716) -> (continuation of Deferred@581519911 after retry RPC after error@580998405) -> (continuation of Deferred@1738327846 after retry RPC after error@945817641) -> (continuation of Deferred@1349551472 after retry RPC after error@2018634502) -> (continuation of Deferred@2046142173 after retry RPC after error@1106040970) -> (continuation of Deferred@367238850 after retry RPC after error@1654480547) -> (continuation of Deferred@906678115 after retry RPC after error@293960096) -> (continuation of Deferred@404349148 after retry RPC after error@276421946) -> (continuation of Deferred@1894279553 after retry RPC after error@1251628613) -> (continuation of Deferred@500318410 after retry RPC after error@1125965383) -> (continuation of Deferred@1888162756 after retry RPC after error@1748716526) -> (continuation of Deferred@1734382131 after retry RPC after error@1110152830) -> (continuation of Deferred@1779869900 after retry RPC after error@1480649307)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.76.190:46649,127.11.76.188:34283,127.11.76.189:35707) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.190:46649(127.11.76.190:46649): Connection refused: /127.11.76.190:46649,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.76.189:35707(127.11.76.189:35707): Connection refused: /127.11.76.189:35707
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.76.189:35707
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)