18:56:02.587 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
18:56:02.602 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task01LnxB/test-tmp
18:56:02.605 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
18:56:02.609 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task01LnxB/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
18:56:02.900 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
18:56:03.081 [DEBUG - main] (MiniKuduCluster.java:195) Response:
18:56:03.082 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
18:56:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.087159 15847 mini_oidc.cc:152] Starting JWKS server
18:56:03.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.092713 15847 webserver.cc:460] Webserver started at http://127.0.0.1:38205/ using document root <none> and password file <none>
18:56:03.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.093263 15847 mini_oidc.cc:189] Starting OIDC Discovery server
18:56:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.095769 15847 webserver.cc:460] Webserver started at http://127.0.0.1:33047/ using document root <none> and password file <none>
18:56:03.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:56:03Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
18:56:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:56:03Z Disabled control of system clock
18:56:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.161394 15847 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:03.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/wal
18:56:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data
18:56:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/logs
18:56:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data/info.pb
18:56:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:03.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:03.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:03.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:03.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:03.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:03.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:03.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:03.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:03.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.254:35303
18:56:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.254
18:56:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:03.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38205/jwks.json with env {}
18:56:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.528465 15859 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.529143 15859 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.529865 15859 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.563133 15859 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.563503 15859 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.563719 15859 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.563905 15859 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.564188 15859 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.564419 15859 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38205/jwks.json
18:56:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.602773 15859 master_runner.cc:386] Master server non-default flags:
18:56:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:03.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/wal
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.254:35303
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38205/jwks.json
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data/info.pb
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.254
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/logs
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.604501 15859 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:03.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:03.607210 15859 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:03.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.633315 15866 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:04.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.633431 15865 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:04.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:03.633314 15868 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:04.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:04.882323 15867 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1248 milliseconds
18:56:04.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.882443 15859 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:56:04.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.884163 15859 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:04.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.887704 15859 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:04.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.889117 15859 hybrid_clock.cc:648] HybridClock initialized: now 1781895364889084 us; error 67 us; skew 500 ppm
18:56:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.890220 15859 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:04.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.903110 15859 webserver.cc:460] Webserver started at http://127.15.121.254:32931/ using document root <none> and password file <none>
18:56:04.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.904304 15859 fs_manager.cc:359] Metadata directory not provided
18:56:04.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.904518 15859 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:04.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.904989 15859 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:04.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.910784 15859 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data/instance:
18:56:04.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37ca0c3b32ce4a98b3e651435459523d"
18:56:04.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:04 on dist-test-slave-6bn0"
18:56:04.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.912285 15859 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/wal/instance:
18:56:04.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37ca0c3b32ce4a98b3e651435459523d"
18:56:04.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:04 on dist-test-slave-6bn0"
18:56:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.921339 15859 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
18:56:04.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.928376 15875 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:04.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.929934 15859 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
18:56:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.930387 15859 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/wal
18:56:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "37ca0c3b32ce4a98b3e651435459523d"
18:56:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:04 on dist-test-slave-6bn0"
18:56:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.930860 15859 fs_report.cc:383] FS layout report
18:56:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/wal
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/wal
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data/data
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:04.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:04.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:04.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.979986 15859 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:04.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.982096 15859 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:04.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:04.982750 15859 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.090767 15859 rpc_server.cc:307] RPC server started. Bound to: 127.15.121.254:35303
18:56:05.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.090868 15926 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.121.254:35303 every 8 connection(s)
18:56:05.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.094396 15859 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/data/info.pb
18:56:05.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.098326 15847 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 15859
18:56:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.098970 15847 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-0/wal/instance
18:56:05.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.103667 15927 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:56:05.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.105151 15847 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:05.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:05.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/wal
18:56:05.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/logs
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data/info.pb
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.253:39365
18:56:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.253
18:56:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:05.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:05.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:05.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38205/jwks.json with env {}
18:56:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.121690 15927 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } has no permanent_uuid. Determining permanent_uuid...
18:56:05.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.159613 15927 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } has no permanent_uuid. Determining permanent_uuid...
18:56:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.162675 15877 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:05.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.168596 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.253:39365: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:05.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.218463 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } attempt: 1
18:56:05.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.223368 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.253:39365: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:05.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.291982 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } attempt: 2
18:56:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.297452 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.253:39365: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:05.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.389045 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } attempt: 3
18:56:05.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.393939 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.253:39365: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.500274 15929 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.500983 15929 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:05.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.501581 15929 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.534572 15929 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.534909 15929 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.535230 15929 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.535585 15929 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.535913 15929 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.536170 15929 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38205/jwks.json
18:56:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.537420 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } attempt: 4
18:56:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.542102 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.253:39365: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.576584 15929 master_runner.cc:386] Master server non-default flags:
18:56:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/wal
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.253:39365
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38205/jwks.json
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data/info.pb
18:56:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.253
18:56:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/logs
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.578392 15929 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.581166 15929 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:05.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.604576 15939 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:05.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.841557 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } attempt: 5
18:56:05.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.847245 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.253:39365: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:06.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.394793 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } attempt: 6
18:56:06.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:06.404243 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.253:39365: Network error: Client connection negotiation failed: client connection to 127.15.121.253:39365: connect: Connection refused (error 111)
18:56:06.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.604586 15941 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:05.605396 15938 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:06.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:05.607844 15929 server_base.cc:1034] running on GCE node
18:56:06.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.853688 15929 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:06.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.857542 15929 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:06.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.859014 15929 hybrid_clock.cc:648] HybridClock initialized: now 1781895366858977 us; error 56 us; skew 500 ppm
18:56:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.860119 15929 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:06.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.868247 15929 webserver.cc:460] Webserver started at http://127.15.121.253:43135/ using document root <none> and password file <none>
18:56:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.869391 15929 fs_manager.cc:359] Metadata directory not provided
18:56:06.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.869602 15929 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:06.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.870064 15929 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.876461 15929 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data/instance:
18:56:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f547517fec9849f583f91ff72e1bbfbd"
18:56:06.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:06 on dist-test-slave-6bn0"
18:56:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.878054 15929 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/wal/instance:
18:56:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f547517fec9849f583f91ff72e1bbfbd"
18:56:06.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:06 on dist-test-slave-6bn0"
18:56:06.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.887132 15929 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.005s
18:56:06.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.893953 15950 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:06.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.895226 15929 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
18:56:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.895740 15929 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/wal
18:56:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f547517fec9849f583f91ff72e1bbfbd"
18:56:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:06 on dist-test-slave-6bn0"
18:56:06.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.896178 15929 fs_report.cc:383] FS layout report
18:56:06.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/wal
18:56:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/wal
18:56:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data/data
18:56:06.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:06.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.950981 15929 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:06.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.953310 15929 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:06.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:06.953954 15929 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:07.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.064456 15929 rpc_server.cc:307] RPC server started. Bound to: 127.15.121.253:39365
18:56:07.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.064586 16001 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.121.253:39365 every 8 connection(s)
18:56:07.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.067935 15929 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/data/info.pb
18:56:07.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.075047 16002 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:56:07.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.075846 15847 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 15929
18:56:07.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.076385 15847 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-1/wal/instance
18:56:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.080013 15847 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/wal
18:56:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data
18:56:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:07.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/logs
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data/info.pb
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.252:36031
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.252
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:07.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:07.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38205/jwks.json with env {}
18:56:07.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.098965 16002 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } has no permanent_uuid. Determining permanent_uuid...
18:56:07.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.143409 16002 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } has no permanent_uuid. Determining permanent_uuid...
18:56:07.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.171607 16002 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } has no permanent_uuid. Determining permanent_uuid...
18:56:07.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.175839 15954 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.183884 16002 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.233628 16002 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 1
18:56:07.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.240113 16002 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.308653 16002 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 2
18:56:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.314523 16002 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.406234 16002 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 3
18:56:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.412060 16002 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.464720 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } attempt: 7
18:56:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.487716 15927 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } has no permanent_uuid. Determining permanent_uuid...
18:56:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.498796 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.555738 16002 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 4
18:56:07.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.557372 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 1
18:56:07.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.563148 16002 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.563899 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.589233 16004 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.589973 16004 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:07.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.590567 16004 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.623207 16004 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.623580 16004 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.623850 16004 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.624140 16004 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.624456 16004 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.624713 16004 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38205/jwks.json
18:56:07.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.645582 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 2
18:56:07.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.651232 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.663107 16004 master_runner.cc:386] Master server non-default flags:
18:56:07.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/wal
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:07.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.252:36031
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38205/jwks.json
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data/info.pb
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.252
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/logs
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:07.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.665033 16004 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:07.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.667506 16004 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.704527 16016 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:07.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.736800 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 3
18:56:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.742657 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.862862 16002 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 5
18:56:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.871121 16002 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:07.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:07.883211 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 4
18:56:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.893116 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:08.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:08.177007 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 5
18:56:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:08.185146 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:08.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:08.418876 16002 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 6
18:56:08.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:08.436375 16002 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:08.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:08.737998 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 6
18:56:08.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:08.755357 15927 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.15.121.252:36031: Network error: Client connection negotiation failed: client connection to 127.15.121.252:36031: connect: Connection refused (error 111)
18:56:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.705250 16017 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:09.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:07.704900 16019 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:09.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:09.145354 16018 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1440 milliseconds
18:56:09.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.145486 16004 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:56:09.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.146973 16004 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:09.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.151943 16004 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.153478 16004 hybrid_clock.cc:648] HybridClock initialized: now 1781895369153395 us; error 105 us; skew 500 ppm
18:56:09.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.154597 16004 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:09.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.165972 16004 webserver.cc:460] Webserver started at http://127.15.121.252:44215/ using document root <none> and password file <none>
18:56:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.167188 16004 fs_manager.cc:359] Metadata directory not provided
18:56:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.167481 16004 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.167948 16004 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:09.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.173743 16004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data/instance:
18:56:09.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c744030279c4d6a8315b7a43e71bb65"
18:56:09.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:09 on dist-test-slave-6bn0"
18:56:09.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.175114 16004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/wal/instance:
18:56:09.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c744030279c4d6a8315b7a43e71bb65"
18:56:09.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:09 on dist-test-slave-6bn0"
18:56:09.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.185871 16004 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.000s sys 0.008s
18:56:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.194540 16030 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.196159 16004 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
18:56:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.196532 16004 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/wal
18:56:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8c744030279c4d6a8315b7a43e71bb65"
18:56:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:09 on dist-test-slave-6bn0"
18:56:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.196962 16004 fs_report.cc:383] FS layout report
18:56:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/wal
18:56:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/wal
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data/data
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:09.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.254875 16004 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.257107 16004 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.257683 16004 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:09.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.444386 16004 rpc_server.cc:307] RPC server started. Bound to: 127.15.121.252:36031
18:56:09.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.444628 16081 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.121.252:36031 every 8 connection(s)
18:56:09.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.450119 16004 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/data/info.pb
18:56:09.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.456761 15847 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16004
18:56:09.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.457366 15847 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/master-2/wal/instance
18:56:09.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.462659 16082 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:56:09.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.492187 16082 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } has no permanent_uuid. Determining permanent_uuid...
18:56:09.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.497125 16002 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 7
18:56:09.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.555202 16082 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } has no permanent_uuid. Determining permanent_uuid...
18:56:09.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.578465 16002 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd: Bootstrap starting.
18:56:09.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.579290 16082 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } has no permanent_uuid. Determining permanent_uuid...
18:56:09.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.593914 16002 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd: Neither blocks nor log segments found. Creating new log.
18:56:09.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.598831 16002 log.cc:824] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd: Log is configured to *not* fsync() on all Append() calls
18:56:09.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.609328 16002 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd: No bootstrap required, opened a new log
18:56:09.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.614696 16082 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: Bootstrap starting.
18:56:09.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.632474 16082 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: Neither blocks nor log segments found. Creating new log.
18:56:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.637909 16082 log.cc:824] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: Log is configured to *not* fsync() on all Append() calls
18:56:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.649912 16082 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: No bootstrap required, opened a new log
18:56:09.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.666626 16002 raft_consensus.cc:348] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.668898 16002 raft_consensus.cc:374] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:56:09.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.669761 16002 raft_consensus.cc:729] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f547517fec9849f583f91ff72e1bbfbd, State: Initialized, Role: FOLLOWER
18:56:09.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.671772 16002 consensus_queue.cc:260] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.678380 16094 sys_catalog.cc:455] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.680351 16094 sys_catalog.cc:458] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: This master's current role is: FOLLOWER
18:56:09.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.681854 16002 sys_catalog.cc:564] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: configured and running, proceeding with master startup.
18:56:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.700623 16082 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.702149 16082 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:56:09.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.702649 16082 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8c744030279c4d6a8315b7a43e71bb65, State: Initialized, Role: FOLLOWER
18:56:09.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.704162 16082 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.712368 16097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:09.715554 15956 tablet.cc:2369] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:56:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.716076 16097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [sys.catalog]: This master's current role is: FOLLOWER
18:56:09.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.719419 16082 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [sys.catalog]: configured and running, proceeding with master startup.
18:56:09.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:09.770373 16036 tablet.cc:2369] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:56:09.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.789167 15927 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } attempt: 7
18:56:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.807636 16119 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
18:56:09.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:09.812705 16112 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:09.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:09.814652 16112 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.837580 16094 raft_consensus.cc:482] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.838488 16094 raft_consensus.cc:504] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.777468 15929 master_runner.cc:186] Error getting master registration for 127.15.121.254:35303: OK, instance_id { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" instance_seqno: 1781895365060334 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:56:09.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.843910 15929 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.847975 16094 leader_election.cc:288] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 37ca0c3b32ce4a98b3e651435459523d (127.15.121.254:35303), 8c744030279c4d6a8315b7a43e71bb65 (127.15.121.252:36031)
18:56:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.849063 16057 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f547517fec9849f583f91ff72e1bbfbd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c744030279c4d6a8315b7a43e71bb65" is_pre_election: true
18:56:09.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.850638 16057 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f547517fec9849f583f91ff72e1bbfbd in term 0.
18:56:09.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.853170 15927 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: Bootstrap starting.
18:56:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.853120 15954 leader_election.cc:302] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8c744030279c4d6a8315b7a43e71bb65, f547517fec9849f583f91ff72e1bbfbd; no voters:
18:56:09.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.854789 16094 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Leader pre-election won for term 1
18:56:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.855299 16094 raft_consensus.cc:482] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
18:56:09.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.855826 16094 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 0 FOLLOWER]: Advancing to term 1
18:56:09.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.872226 15927 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: Neither blocks nor log segments found. Creating new log.
18:56:09.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.875886 16094 raft_consensus.cc:504] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.878695 15927 log.cc:824] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: Log is configured to *not* fsync() on all Append() calls
18:56:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.895635 16057 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f547517fec9849f583f91ff72e1bbfbd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8c744030279c4d6a8315b7a43e71bb65"
18:56:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.896000 16094 leader_election.cc:288] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [CANDIDATE]: Term 1 election: Requested vote from peers 37ca0c3b32ce4a98b3e651435459523d (127.15.121.254:35303), 8c744030279c4d6a8315b7a43e71bb65 (127.15.121.252:36031)
18:56:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.896569 16057 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [term 0 FOLLOWER]: Advancing to term 1
18:56:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.908443 15927 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: No bootstrap required, opened a new log
18:56:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.908812 16057 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f547517fec9849f583f91ff72e1bbfbd in term 1.
18:56:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.919940 15954 leader_election.cc:302] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8c744030279c4d6a8315b7a43e71bb65, f547517fec9849f583f91ff72e1bbfbd; no voters:
18:56:09.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.921990 16094 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 1 FOLLOWER]: Leader election won for term 1
18:56:09.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.814826 16004 master_runner.cc:186] Error getting master registration for 127.15.121.254:35303: OK, instance_id { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" instance_seqno: 1781895365060334 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:56:09.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.942830 16004 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:09.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.944875 16094 raft_consensus.cc:686] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [term 1 LEADER]: Becoming Leader. State: Replica: f547517fec9849f583f91ff72e1bbfbd, State: Running, Role: LEADER
18:56:09.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.946600 16094 consensus_queue.cc:237] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.962994 15927 raft_consensus.cc:348] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.964546 15927 raft_consensus.cc:374] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:56:09.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.966150 15927 raft_consensus.cc:729] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 37ca0c3b32ce4a98b3e651435459523d, State: Initialized, Role: FOLLOWER
18:56:09.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.968904 15927 consensus_queue.cc:260] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } }
18:56:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.976774 16121 sys_catalog.cc:455] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: SysCatalogTable state changed. Reason: New leader f547517fec9849f583f91ff72e1bbfbd. Latest consensus state: current_term: 1 leader_uuid: "f547517fec9849f583f91ff72e1bbfbd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:09.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.978368 16121 sys_catalog.cc:458] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: This master's current role is: LEADER
18:56:09.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.977602 16123 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:09.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.979108 16123 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [sys.catalog]: This master's current role is: FOLLOWER
18:56:09.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.981869 15927 sys_catalog.cc:564] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [sys.catalog]: configured and running, proceeding with master startup.
18:56:09.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.982421 15902 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f547517fec9849f583f91ff72e1bbfbd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "37ca0c3b32ce4a98b3e651435459523d" is_pre_election: true
18:56:09.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.984076 15902 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f547517fec9849f583f91ff72e1bbfbd in term 0.
18:56:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.984529 15901 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f547517fec9849f583f91ff72e1bbfbd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "37ca0c3b32ce4a98b3e651435459523d"
18:56:09.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:09.986116 15901 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [term 0 FOLLOWER]: Advancing to term 1
18:56:09.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:09.997385 15881 tablet.cc:2369] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:56:10.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.004031 15901 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f547517fec9849f583f91ff72e1bbfbd in term 1.
18:56:10.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.012512 16124 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
18:56:10.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.030340 16124 catalog_manager.cc:1484] Initializing Kudu cluster ID...
18:56:10.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.076555 16057 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [term 1 FOLLOWER]: Refusing update from remote peer f547517fec9849f583f91ff72e1bbfbd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:56:10.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.080089 15901 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [term 1 FOLLOWER]: Refusing update from remote peer f547517fec9849f583f91ff72e1bbfbd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
18:56:10.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.081972 16094 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [LEADER]: Connected to new peer: Peer: permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
18:56:10.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.085031 16094 consensus_queue.cc:1035] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [LEADER]: Connected to new peer: Peer: permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
18:56:10.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.118643 16097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f547517fec9849f583f91ff72e1bbfbd. Latest consensus state: current_term: 1 leader_uuid: "f547517fec9849f583f91ff72e1bbfbd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.119920 16097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [sys.catalog]: This master's current role is: FOLLOWER
18:56:10.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.136138 16097 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f547517fec9849f583f91ff72e1bbfbd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:10.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.137271 16097 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65 [sys.catalog]: This master's current role is: FOLLOWER
18:56:10.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.153541 16121 sys_catalog.cc:455] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f547517fec9849f583f91ff72e1bbfbd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.155373 16121 sys_catalog.cc:458] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: This master's current role is: LEADER
18:56:10.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.160389 16124 catalog_manager.cc:1347] Generated new cluster ID: c632fc3658a042618ed004ba9a590abb
18:56:10.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.160817 16124 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
18:56:10.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.168839 16123 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f547517fec9849f583f91ff72e1bbfbd. Latest consensus state: current_term: 1 leader_uuid: "f547517fec9849f583f91ff72e1bbfbd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:10.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.175595 16123 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [sys.catalog]: This master's current role is: FOLLOWER
18:56:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.215992 16140 mvcc.cc:205] Tried to move back new op lower bound from 7298643435797708800 to 7298643435317993472. Current Snapshot: MvccSnapshot[applied={T|T < 7298643435797708800}]
18:56:10.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.206559 16141 sys_catalog.cc:455] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f547517fec9849f583f91ff72e1bbfbd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:10.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.218757 16141 sys_catalog.cc:458] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd [sys.catalog]: This master's current role is: LEADER
18:56:10.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.239431 16123 sys_catalog.cc:455] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f547517fec9849f583f91ff72e1bbfbd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "37ca0c3b32ce4a98b3e651435459523d" member_type: VOTER last_known_addr { host: "127.15.121.254" port: 35303 } } peers { permanent_uuid: "f547517fec9849f583f91ff72e1bbfbd" member_type: VOTER last_known_addr { host: "127.15.121.253" port: 39365 } } peers { permanent_uuid: "8c744030279c4d6a8315b7a43e71bb65" member_type: VOTER last_known_addr { host: "127.15.121.252" port: 36031 } } }
18:56:10.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.240590 16123 sys_catalog.cc:458] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d [sys.catalog]: This master's current role is: FOLLOWER
18:56:10.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:10.250607 16143 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:10.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:10.251125 16143 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:10.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.303445 16124 catalog_manager.cc:1370] Generated new certificate authority record
18:56:10.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.307936 16124 catalog_manager.cc:1504] Loading token signing keys...
18:56:10.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.357786 16124 catalog_manager.cc:5868] T 00000000000000000000000000000000 P f547517fec9849f583f91ff72e1bbfbd: Generated new TSK 0
18:56:10.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.360880 16124 catalog_manager.cc:1514] Initializing in-progress tserver states...
18:56:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.386574 15847 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/wal
18:56:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data
18:56:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/logs
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data/info.pb
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:10.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.193:0
18:56:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.121.193
18:56:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.193
18:56:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:10.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:10.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:56:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.845425 16119 catalog_manager.cc:1259] Loaded cluster ID: c632fc3658a042618ed004ba9a590abb
18:56:10.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.846232 16119 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: loading cluster ID for follower catalog manager: success
18:56:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.863184 16119 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: acquiring CA information for follower catalog manager: success
18:56:10.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:10.889508 16119 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8c744030279c4d6a8315b7a43e71bb65: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:56:11.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.100107 16146 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:11.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.101104 16146 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.102268 16146 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:11.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.178020 16146 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:11.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.179569 16146 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.121.193
18:56:11.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:11.259356 16143 catalog_manager.cc:1259] Loaded cluster ID: c632fc3658a042618ed004ba9a590abb
18:56:11.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:11.260212 16143 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: loading cluster ID for follower catalog manager: success
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:11.264745 16146 tablet_server_runner.cc:78] Tablet server non-default flags:
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/wal
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.193:0
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data/info.pb
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.193
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.121.193
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/logs
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:11.266800 16146 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:11.269932 16146 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:11.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:11.294454 16143 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: acquiring CA information for follower catalog manager: success
18:56:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:11.335873 16143 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 37ca0c3b32ce4a98b3e651435459523d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
18:56:11.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.358111 16154 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:13.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:13.629792 15954 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.15.121.252:36031, user_credentials={real_user=slave}} blocked reactor thread for 65197us
18:56:13.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.716759 16146 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:56:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.360962 16155 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:11.372211 16157 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:13.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:13.716051 16156 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
18:56:13.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.742020 16146 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:13.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.766819 16146 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.775677 16146 hybrid_clock.cc:648] HybridClock initialized: now 1781895373775468 us; error 456 us; skew 500 ppm
18:56:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.782604 16146 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.855669 16146 webserver.cc:460] Webserver started at http://127.15.121.193:41117/ using document root <none> and password file <none>
18:56:13.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.862488 16146 fs_manager.cc:359] Metadata directory not provided
18:56:13.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.866690 16146 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.868899 16146 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:13.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.909271 16146 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data/instance:
18:56:13.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80d7a3357368409e93d13d9c2ebdb1aa"
18:56:13.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:13 on dist-test-slave-6bn0"
18:56:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.917652 16146 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/wal/instance:
18:56:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80d7a3357368409e93d13d9c2ebdb1aa"
18:56:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:13 on dist-test-slave-6bn0"
18:56:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:13.986292 16146 fs_manager.cc:691] Time spent creating directory manager: real 0.063s user 0.036s sys 0.011s
18:56:14.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.026400 16168 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:14.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.040896 16146 fs_manager.cc:725] Time spent opening block manager: real 0.033s user 0.007s sys 0.012s
18:56:14.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.045270 16146 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/wal
18:56:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80d7a3357368409e93d13d9c2ebdb1aa"
18:56:14.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:13 on dist-test-slave-6bn0"
18:56:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.049139 16146 fs_report.cc:383] FS layout report
18:56:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:14.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/wal
18:56:14.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/wal
18:56:14.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data/data
18:56:14.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:14.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:14.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:14.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.494885 16146 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:14.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.505800 16146 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.507733 16146 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.548044 16146 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:56:14.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.618920 16146 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:56:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.620308 16146 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
18:56:14.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.621230 16146 ts_tablet_manager.cc:610] Registered 0 tablets
18:56:14.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:14.622273 16146 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s user 0.002s sys 0.000s
18:56:15.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.273185 16146 rpc_server.cc:307] RPC server started. Bound to: 127.15.121.193:35671
18:56:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.273465 16284 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.121.193:35671 every 8 connection(s)
18:56:15.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.277348 16146 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/data/info.pb
18:56:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.284313 15847 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16146
18:56:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.285418 15847 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-0/wal/instance
18:56:15.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.308225 15847 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/wal
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/logs
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data/info.pb
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:15.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.194:0
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.121.194
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.194
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:56:15.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.387936 16287 heartbeater.cc:347] Connected to a master server at 127.15.121.254:35303
18:56:15.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.389115 16287 heartbeater.cc:464] Registering TS with master...
18:56:15.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.390933 16287 heartbeater.cc:511] Master 127.15.121.254:35303 requested a full tablet report, sending...
18:56:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.395021 16286 heartbeater.cc:347] Connected to a master server at 127.15.121.253:39365
18:56:15.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.395507 16286 heartbeater.cc:464] Registering TS with master...
18:56:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.396961 16286 heartbeater.cc:511] Master 127.15.121.253:39365 requested a full tablet report, sending...
18:56:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.398021 15891 ts_manager.cc:194] Registered new tserver with Master: 80d7a3357368409e93d13d9c2ebdb1aa (127.15.121.193:35671)
18:56:15.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.400770 15967 ts_manager.cc:194] Registered new tserver with Master: 80d7a3357368409e93d13d9c2ebdb1aa (127.15.121.193:35671)
18:56:15.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.406487 15967 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.121.193:53851
18:56:15.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.408077 16285 heartbeater.cc:347] Connected to a master server at 127.15.121.252:36031
18:56:15.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.408548 16285 heartbeater.cc:464] Registering TS with master...
18:56:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.409934 16285 heartbeater.cc:511] Master 127.15.121.252:36031 requested a full tablet report, sending...
18:56:15.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.415169 16047 ts_manager.cc:194] Registered new tserver with Master: 80d7a3357368409e93d13d9c2ebdb1aa (127.15.121.193:35671)
18:56:15.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.752126 16291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:15.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.752791 16291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:15.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.753508 16291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:15.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.788718 16291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:15.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.789827 16291 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.121.194
18:56:15.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.828787 16291 tablet_server_runner.cc:78] Tablet server non-default flags:
18:56:15.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:15.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:15.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/wal
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.194:0
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data/info.pb
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.194
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:15.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.121.194
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/logs
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:15.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.830715 16291 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:15.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.833354 16291 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:15.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.858639 16306 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:16.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:16.414439 16286 heartbeater.cc:503] Master 127.15.121.253:39365 was elected leader, sending a full tablet report...
18:56:17.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.858683 16303 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:17.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:15.858671 16304 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:17.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:15.861317 16291 server_base.cc:1034] running on GCE node
18:56:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.177265 16291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:17.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.180994 16291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:17.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.182596 16291 hybrid_clock.cc:648] HybridClock initialized: now 1781895377182545 us; error 67 us; skew 500 ppm
18:56:17.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.183858 16291 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:17.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.197471 16291 webserver.cc:460] Webserver started at http://127.15.121.194:44367/ using document root <none> and password file <none>
18:56:17.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.198661 16291 fs_manager.cc:359] Metadata directory not provided
18:56:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.198869 16291 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:17.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.199373 16291 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.205127 16291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data/instance:
18:56:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f75e5257ec8148d8ba6f1b76bd6cdc2b"
18:56:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:17 on dist-test-slave-6bn0"
18:56:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.206315 16291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/wal/instance:
18:56:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f75e5257ec8148d8ba6f1b76bd6cdc2b"
18:56:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:17 on dist-test-slave-6bn0"
18:56:17.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.216446 16291 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
18:56:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.225442 16313 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.226786 16291 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
18:56:17.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.227195 16291 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/wal
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f75e5257ec8148d8ba6f1b76bd6cdc2b"
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:17 on dist-test-slave-6bn0"
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.227666 16291 fs_report.cc:383] FS layout report
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/wal
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/wal
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data/data
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:17.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.283237 16291 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.285620 16291 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:17.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.286372 16291 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.291174 16291 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:56:17.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.299492 16291 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:56:17.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.299728 16291 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
18:56:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.299970 16291 ts_tablet_manager.cc:610] Registered 0 tablets
18:56:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.300110 16291 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
18:56:17.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.631011 16291 rpc_server.cc:307] RPC server started. Bound to: 127.15.121.194:43743
18:56:17.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.631145 16426 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.121.194:43743 every 8 connection(s)
18:56:17.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.633857 16291 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/data/info.pb
18:56:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.641390 15847 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16291
18:56:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.642025 15847 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-1/wal/instance
18:56:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.680969 15847 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/wal
18:56:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data
18:56:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/logs
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data/info.pb
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.195:0
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.121.195
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.195
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:17.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:17.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:17.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:17.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:17.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
18:56:17.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.702291 16429 heartbeater.cc:347] Connected to a master server at 127.15.121.254:35303
18:56:17.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.702896 16429 heartbeater.cc:464] Registering TS with master...
18:56:17.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.704834 16429 heartbeater.cc:511] Master 127.15.121.254:35303 requested a full tablet report, sending...
18:56:17.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.706218 16427 heartbeater.cc:347] Connected to a master server at 127.15.121.252:36031
18:56:17.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.706605 16427 heartbeater.cc:464] Registering TS with master...
18:56:17.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.708210 16427 heartbeater.cc:511] Master 127.15.121.252:36031 requested a full tablet report, sending...
18:56:17.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.709949 15891 ts_manager.cc:194] Registered new tserver with Master: f75e5257ec8148d8ba6f1b76bd6cdc2b (127.15.121.194:43743)
18:56:17.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.710764 16428 heartbeater.cc:347] Connected to a master server at 127.15.121.253:39365
18:56:17.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.711208 16428 heartbeater.cc:464] Registering TS with master...
18:56:17.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.711747 16047 ts_manager.cc:194] Registered new tserver with Master: f75e5257ec8148d8ba6f1b76bd6cdc2b (127.15.121.194:43743)
18:56:17.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.712394 16428 heartbeater.cc:511] Master 127.15.121.253:39365 requested a full tablet report, sending...
18:56:17.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.715130 15967 ts_manager.cc:194] Registered new tserver with Master: f75e5257ec8148d8ba6f1b76bd6cdc2b (127.15.121.194:43743)
18:56:17.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:17.716938 15967 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.121.194:56017
18:56:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.163954 16437 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:18.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.164582 16437 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.165019 16437 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:18.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.197139 16437 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:18.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.198078 16437 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.15.121.195
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:18.235545 16437 tablet_server_runner.cc:78] Tablet server non-default flags:
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.15.121.212:35091
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/wal
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.15.121.195:0
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data/info.pb
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.15.121.195
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.15.121.254:35303,127.15.121.253:39365,127.15.121.252:36031
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.15.121.195
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/logs
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:18.237174 16437 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:18.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:18.239181 16437 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:18.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.272598 16443 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:18.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:18.721405 16428 heartbeater.cc:503] Master 127.15.121.253:39365 was elected leader, sending a full tablet report...
18:56:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.276077 16446 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:18.276002 16444 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:19.679986 16445 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1407 milliseconds
18:56:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.680231 16437 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:56:19.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.681526 16437 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:19.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.685258 16437 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:19.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.686801 16437 hybrid_clock.cc:648] HybridClock initialized: now 1781895379686780 us; error 57 us; skew 500 ppm
18:56:19.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.687857 16437 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:19.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.705447 16437 webserver.cc:460] Webserver started at http://127.15.121.195:41947/ using document root <none> and password file <none>
18:56:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.706732 16437 fs_manager.cc:359] Metadata directory not provided
18:56:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.707019 16437 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.707624 16437 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:19.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.713675 16437 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data/instance:
18:56:19.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e3b71887d744185ab8ab8a44163d1e7"
18:56:19.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:19 on dist-test-slave-6bn0"
18:56:19.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.715147 16437 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/wal/instance:
18:56:19.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e3b71887d744185ab8ab8a44163d1e7"
18:56:19.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:19 on dist-test-slave-6bn0"
18:56:19.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.728372 16437 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.005s sys 0.005s
18:56:19.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.737771 16454 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:19.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.739243 16437 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.001s
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.739778 16437 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/wal
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e3b71887d744185ab8ab8a44163d1e7"
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:19 on dist-test-slave-6bn0"
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.740131 16437 fs_report.cc:383] FS layout report
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/wal
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/wal
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data/data
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.785194 16437 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:19.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.787376 16437 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:19.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.788033 16437 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:19.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.792964 16437 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
18:56:19.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.799903 16437 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
18:56:19.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.800212 16437 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
18:56:19.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.800541 16437 ts_tablet_manager.cc:610] Registered 0 tablets
18:56:19.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:19.800724 16437 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
18:56:20.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.206795 16437 rpc_server.cc:307] RPC server started. Bound to: 127.15.121.195:35171
18:56:20.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.207008 16566 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.15.121.195:35171 every 8 connection(s)
18:56:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.209895 16437 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/data/info.pb
18:56:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.212080 15847 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16437
18:56:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.212714 15847 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster757452486596851465/ts-2/wal/instance
18:56:20.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.290297 16568 heartbeater.cc:347] Connected to a master server at 127.15.121.253:39365
18:56:20.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.291085 16568 heartbeater.cc:464] Registering TS with master...
18:56:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.293262 16567 heartbeater.cc:347] Connected to a master server at 127.15.121.252:36031
18:56:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.293797 16567 heartbeater.cc:464] Registering TS with master...
18:56:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.294811 16567 heartbeater.cc:511] Master 127.15.121.252:36031 requested a full tablet report, sending...
18:56:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.293244 16568 heartbeater.cc:511] Master 127.15.121.253:39365 requested a full tablet report, sending...
18:56:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.295914 16570 heartbeater.cc:347] Connected to a master server at 127.15.121.254:35303
18:56:20.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.296346 16570 heartbeater.cc:464] Registering TS with master...
18:56:20.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.297880 16570 heartbeater.cc:511] Master 127.15.121.254:35303 requested a full tablet report, sending...
18:56:20.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.300345 16047 ts_manager.cc:194] Registered new tserver with Master: 5e3b71887d744185ab8ab8a44163d1e7 (127.15.121.195:35171)
18:56:20.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.301072 15967 ts_manager.cc:194] Registered new tserver with Master: 5e3b71887d744185ab8ab8a44163d1e7 (127.15.121.195:35171)
18:56:20.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.302054 15891 ts_manager.cc:194] Registered new tserver with Master: 5e3b71887d744185ab8ab8a44163d1e7 (127.15.121.195:35171)
18:56:20.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.304523 15967 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.15.121.195:52079
18:56:20.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:20.315238 15847 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
18:56:20.334 [DEBUG - main] (MiniKuduCluster.java:195) Response:
18:56:20.336 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
18:56:20.348 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.15.121.254"
port: 35303
}
bound_http_address {
host: "127.15.121.254"
port: 32931
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.15.121.253"
port: 39365
}
bound_http_address {
host: "127.15.121.253"
port: 43135
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.15.121.252"
port: 36031
}
bound_http_address {
host: "127.15.121.252"
port: 44215
}
}
}
18:56:20.355 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
18:56:20.358 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.15.121.193"
port: 35671
}
bound_http_address {
host: "127.15.121.193"
port: 41117
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.15.121.194"
port: 43743
}
bound_http_address {
host: "127.15.121.194"
port: 44367
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.15.121.195"
port: 35171
}
bound_http_address {
host: "127.15.121.195"
port: 41947
}
}
}
18:56:20.359 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
18:56:20.573 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
18:56:20.741 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task01LnxB/test-tmp
18:56:20.742 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
18:56:20.742 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task01LnxB/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
18:56:20.746 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
18:56:21.214 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:21.214 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
18:56:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:21.218086 16578 mini_oidc.cc:152] Starting JWKS server
18:56:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:21.223131 16578 webserver.cc:460] Webserver started at http://127.0.0.1:39769/ using document root <none> and password file <none>
18:56:21.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:21.223721 16578 mini_oidc.cc:189] Starting OIDC Discovery server
18:56:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:21.225667 16578 webserver.cc:460] Webserver started at http://127.0.0.1:44051/ using document root <none> and password file <none>
18:56:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
18:56:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/principal' for realm 'KRBTEST.COM',
18:56:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
18:56:21.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:21.311089 16568 heartbeater.cc:503] Master 127.15.121.253:39365 was elected leader, sending a full tablet report...
18:56:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:21 dist-test-slave-6bn0 krb5kdc[16591](info): setting up network...
18:56:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
18:56:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:21 dist-test-slave-6bn0 krb5kdc[16591](info): set up 2 sockets
18:56:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:21 dist-test-slave-6bn0 krb5kdc[16591](info): commencing operation
18:56:21.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
18:56:22.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:22.398510 16578 mini_kdc.cc:121] Time spent starting KDC: real 1.142s user 0.002s sys 0.035s
18:56:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
18:56:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:56:22.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
18:56:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
18:56:22.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:56:22.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
18:56:22.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
18:56:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:56:22.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
18:56:22.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
18:56:22.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/test-user.keytab.
18:56:22.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/test-user.keytab.
18:56:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:22 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:22.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
18:56:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:56:22Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
18:56:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:56:22Z Disabled control of system clock
18:56:22.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.48.190@KRBTEST.COM; defaulting to no policy
18:56:22.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:56:22.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.48.190@KRBTEST.COM" created.
18:56:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:56:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.48.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.190.keytab.
18:56:22.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.48.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.190.keytab.
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:22.640580 16578 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/logs
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/info.pb
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.190:40049
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.190
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.190.keytab
18:56:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.190
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:56:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.128324 16604 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.129276 16604 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:23.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.130249 16604 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:23.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.164975 16604 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:56:23.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.165237 16604 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:23.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.165510 16604 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:23.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.165709 16604 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:23.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.165895 16604 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:23.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.166181 16604 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:23.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.166406 16604 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39769/jwks.json
18:56:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:23.215416 16604 master_runner.cc:386] Master server non-default flags:
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.190.keytab
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.190
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.190:40049
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/info.pb
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.190
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/logs
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:23.216931 16604 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:23.218137 16604 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.248773 16612 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.252689 16611 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:24.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:23.250233 16614 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:24.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:24.639885 16613 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1392 milliseconds
18:56:24.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.640086 16604 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:56:24.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.641925 16604 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:24.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.646803 16604 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:24.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.648553 16604 hybrid_clock.cc:648] HybridClock initialized: now 1781895384648521 us; error 53 us; skew 500 ppm
18:56:24.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.649660 16604 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:24.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:24 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895384, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:24.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.659744 16604 init.cc:377] Logged in from keytab as kudu/127.16.48.190@KRBTEST.COM (short username kudu)
18:56:24.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.669220 16604 webserver.cc:460] Webserver started at http://127.16.48.190:42119/ using document root <none> and password file <none>
18:56:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.670408 16604 fs_manager.cc:359] Metadata directory not provided
18:56:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.670625 16604 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.671613 16604 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.678295 16604 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/instance:
18:56:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7168256b087346a8ad0605d090e5142b"
18:56:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:24 on dist-test-slave-6bn0"
18:56:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.679744 16604 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal/instance:
18:56:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7168256b087346a8ad0605d090e5142b"
18:56:24.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:24 on dist-test-slave-6bn0"
18:56:24.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.694315 16604 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.011s sys 0.001s
18:56:24.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.703718 16623 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:24.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.705538 16604 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.007s sys 0.001s
18:56:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.706185 16604 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7168256b087346a8ad0605d090e5142b"
18:56:24.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:24 on dist-test-slave-6bn0"
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.706673 16604 fs_report.cc:383] FS layout report
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/data
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:24.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:24.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.760730 16604 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:24.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.768106 16604 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:24.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.768704 16604 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.961158 16604 rpc_server.cc:307] RPC server started. Bound to: 127.16.48.190:40049
18:56:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.961335 16674 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.48.190:40049 every 8 connection(s)
18:56:24.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.964903 16604 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/info.pb
18:56:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.971724 16578 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16604
18:56:24.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.972398 16578 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal/instance
18:56:24.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:24.976329 16675 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:56:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.48.189@KRBTEST.COM; defaulting to no policy
18:56:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.001814 16675 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } has no permanent_uuid. Determining permanent_uuid...
18:56:25.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:56:25.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.48.189@KRBTEST.COM" created.
18:56:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:56:25.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.48.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.189.keytab.
18:56:25.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.48.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.189.keytab.
18:56:25.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.049221 16578 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:25.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/logs
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/info.pb
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.189:36721
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.189
18:56:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.189.keytab
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.189
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:25.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:56:25.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:25 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895384, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.190@KRBTEST.COM for kudu/127.16.48.190@KRBTEST.COM
18:56:25.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.072338 16675 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } has no permanent_uuid. Determining permanent_uuid...
18:56:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.076290 16626 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:25.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.099884 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.189:36721: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:25.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.149513 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } attempt: 1
18:56:25.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.154235 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.189:36721: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:25.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.222754 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } attempt: 2
18:56:25.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.227936 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.189:36721: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.319482 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } attempt: 3
18:56:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.325392 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.189:36721: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:25.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.468873 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } attempt: 4
18:56:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.473886 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.189:36721: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.539017 16681 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.539726 16681 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:25.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.540308 16681 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.572377 16681 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:56:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.572620 16681 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.572885 16681 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.573076 16681 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.573261 16681 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.573447 16681 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.573585 16681 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39769/jwks.json
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.610662 16681 master_runner.cc:386] Master server non-default flags:
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.189.keytab
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.189
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.189:36721
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:25.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/info.pb
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.189
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/logs
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.612205 16681 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:25.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.613507 16681 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.642347 16690 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:25.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.773362 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } attempt: 5
18:56:25.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.778486 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.189:36721: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:26.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:26.325897 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } attempt: 6
18:56:26.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:26.342005 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.189:36721: Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.644223 16689 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:27.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:25.645938 16681 server_base.cc:1034] running on GCE node
18:56:27.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:25.647171 16692 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.024188 16681 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.027647 16681 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:27.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.029043 16681 hybrid_clock.cc:648] HybridClock initialized: now 1781895387029031 us; error 71 us; skew 500 ppm
18:56:27.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.030107 16681 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:27.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:27 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895387, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:27.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.039268 16681 init.cc:377] Logged in from keytab as kudu/127.16.48.189@KRBTEST.COM (short username kudu)
18:56:27.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.053124 16681 webserver.cc:460] Webserver started at http://127.16.48.189:35177/ using document root <none> and password file <none>
18:56:27.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.055518 16681 fs_manager.cc:359] Metadata directory not provided
18:56:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.055887 16681 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.056480 16681 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.063691 16681 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/instance:
18:56:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "903421a259a74750a539cb85f24ef279"
18:56:27.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:27 on dist-test-slave-6bn0"
18:56:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.064919 16681 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal/instance:
18:56:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "903421a259a74750a539cb85f24ef279"
18:56:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:27 on dist-test-slave-6bn0"
18:56:27.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.076522 16681 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.002s sys 0.009s
18:56:27.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.087304 16704 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:27.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.089260 16681 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.002s sys 0.006s
18:56:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.089747 16681 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "903421a259a74750a539cb85f24ef279"
18:56:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:27 on dist-test-slave-6bn0"
18:56:27.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.090248 16681 fs_report.cc:383] FS layout report
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/data
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:27.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:27.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.156731 16681 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:27.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.164414 16681 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:27.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.164981 16681 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:27.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.372095 16681 rpc_server.cc:307] RPC server started. Bound to: 127.16.48.189:36721
18:56:27.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.372398 16755 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.48.189:36721 every 8 connection(s)
18:56:27.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.375478 16681 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/info.pb
18:56:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.380540 16578 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16681
18:56:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.381470 16578 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal/instance
18:56:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.386860 16756 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:56:27.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.402541 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } attempt: 7
18:56:27.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.48.188@KRBTEST.COM; defaulting to no policy
18:56:27.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:56:27.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.48.188@KRBTEST.COM" created.
18:56:27.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.406764 16756 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } has no permanent_uuid. Determining permanent_uuid...
18:56:27.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:27 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895384, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.190@KRBTEST.COM for kudu/127.16.48.189@KRBTEST.COM
18:56:27.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:27 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895387, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.189@KRBTEST.COM for kudu/127.16.48.190@KRBTEST.COM
18:56:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
18:56:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.48.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.188.keytab.
18:56:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.48.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.188.keytab.
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.460952 16578 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:27.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:27.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/logs
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/info.pb
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.188:34995
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.188
18:56:27.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.188.keytab
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.188
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:27.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:56:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.478626 16756 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } has no permanent_uuid. Determining permanent_uuid...
18:56:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.484344 16675 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } has no permanent_uuid. Determining permanent_uuid...
18:56:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.499006 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:27 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895387, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.189@KRBTEST.COM for kudu/127.16.48.189@KRBTEST.COM
18:56:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.515882 16756 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } has no permanent_uuid. Determining permanent_uuid...
18:56:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.519892 16708 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.528539 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.557559 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 1
18:56:27.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.562793 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.578253 16756 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 1
18:56:27.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.584106 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.644369 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 2
18:56:27.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.649785 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.652688 16756 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 2
18:56:27.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.658035 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.735447 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 3
18:56:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.742120 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.749701 16756 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 3
18:56:27.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.756079 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.882571 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 4
18:56:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.888041 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:27.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:27.899428 16756 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 4
18:56:27.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:27.904304 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.020401 16764 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.021405 16764 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:28.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.022408 16764 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.057492 16764 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:56:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.057740 16764 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.058018 16764 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.058239 16764 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.058437 16764 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.058624 16764 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.058780 16764 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39769/jwks.json
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:28.102447 16764 master_runner.cc:386] Master server non-default flags:
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.188.keytab
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.188
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.188:34995
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/info.pb
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.188
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:28.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/logs
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:28.104247 16764 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:28.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:28.105847 16764 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:28.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.140640 16773 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:28.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:28.171541 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 5
18:56:28.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.178280 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:28.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:28.203910 16756 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 5
18:56:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.210248 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:28.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:28.730965 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 6
18:56:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.743065 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:28.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:28.757766 16756 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 6
18:56:28.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.769448 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:28.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.991789 16078 debug-util.cc:398] Leaking SignalData structure 0x7b0800072c60 after lost signal to thread 16011
18:56:28.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.992923 16078 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba000 after lost signal to thread 16081
18:56:29.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.776672 16675 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 7
18:56:29.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:29.812067 16675 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:29.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.830004 16756 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } attempt: 7
18:56:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:29.528927 16772 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 16764
18:56:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:29.836985 16772 kernel_stack_watchdog.cc:198] Thread 16764 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 389ms:
18:56:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
18:56:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
18:56:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
18:56:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
18:56:29.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:28.148790 16775 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:29.846117 16764 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.710s user 0.588s sys 1.073s
18:56:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:29.846549 16764 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.711s user 0.588s sys 1.073s
18:56:29.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.851305 16764 server_base.cc:1034] running on GCE node
18:56:29.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:29.852357 16788 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:29.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.854080 16764 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:29.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:29.856485 16756 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.48.188:34995: Network error: Client connection negotiation failed: client connection to 127.16.48.188:34995: connect: Connection refused (error 111)
18:56:29.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.858846 16764 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:29.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.860319 16764 hybrid_clock.cc:648] HybridClock initialized: now 1781895389860251 us; error 59 us; skew 500 ppm
18:56:29.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.861327 16764 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:29 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895389, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.875007 16764 init.cc:377] Logged in from keytab as kudu/127.16.48.188@KRBTEST.COM (short username kudu)
18:56:29.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.887121 16764 webserver.cc:460] Webserver started at http://127.16.48.188:44779/ using document root <none> and password file <none>
18:56:29.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.888598 16764 fs_manager.cc:359] Metadata directory not provided
18:56:29.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.888862 16764 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:29.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.889408 16764 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
18:56:29.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.895262 16764 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/instance:
18:56:29.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "743ebe9d3ff74b49a3f978aed51061ba"
18:56:29.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:29 on dist-test-slave-6bn0"
18:56:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.896667 16764 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal/instance:
18:56:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "743ebe9d3ff74b49a3f978aed51061ba"
18:56:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:29 on dist-test-slave-6bn0"
18:56:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.908898 16764 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.006s sys 0.006s
18:56:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.925855 16794 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.928316 16764 fs_manager.cc:725] Time spent opening block manager: real 0.014s user 0.008s sys 0.002s
18:56:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.929035 16764 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "743ebe9d3ff74b49a3f978aed51061ba"
18:56:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:29 on dist-test-slave-6bn0"
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.929780 16764 fs_report.cc:383] FS layout report
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/data
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:29.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:29.998170 16764 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:30.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.008421 16764 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.009009 16764 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:30.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.300140 16764 rpc_server.cc:307] RPC server started. Bound to: 127.16.48.188:34995
18:56:30.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.300298 16845 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.48.188:34995 every 8 connection(s)
18:56:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.303746 16764 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/info.pb
18:56:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.312987 16578 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16764
18:56:30.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.313565 16578 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal/instance
18:56:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.330653 16846 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
18:56:30.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.370496 16846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } has no permanent_uuid. Determining permanent_uuid...
18:56:30.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895389, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.188@KRBTEST.COM for kudu/127.16.48.190@KRBTEST.COM
18:56:30.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.48.188@KRBTEST.COM
18:56:30.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.441421 16846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } has no permanent_uuid. Determining permanent_uuid...
18:56:30.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895389, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.188@KRBTEST.COM for kudu/127.16.48.189@KRBTEST.COM
18:56:30.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.48.189@KRBTEST.COM
18:56:30.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.490142 16846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } has no permanent_uuid. Determining permanent_uuid...
18:56:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895389, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.188@KRBTEST.COM for kudu/127.16.48.188@KRBTEST.COM
18:56:30.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.48.190@KRBTEST.COM
18:56:30.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.547755 16846 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Bootstrap starting.
18:56:30.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.555020 16578 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
18:56:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.561009 16846 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Neither blocks nor log segments found. Creating new log.
18:56:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.565551 16846 log.cc:824] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Log is configured to *not* fsync() on all Append() calls
18:56:30.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:30.570 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
18:56:30.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.576305 16846 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: No bootstrap required, opened a new log
18:56:30.589 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
18:56:30.590 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
18:56:30.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.48.190"
port: 40049
}
bound_http_address {
host: "127.16.48.190"
port: 42119
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.48.189"
port: 36721
}
bound_http_address {
host: "127.16.48.189"
port: 35177
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.48.188"
port: 34995
}
bound_http_address {
host: "127.16.48.188"
port: 44779
}
}
}
18:56:30.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
18:56:30.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:30.600 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
18:56:30.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
18:56:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.624375 16846 raft_consensus.cc:348] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.625622 16846 raft_consensus.cc:374] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:56:30.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.626045 16846 raft_consensus.cc:729] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 743ebe9d3ff74b49a3f978aed51061ba, State: Initialized, Role: FOLLOWER
18:56:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.627408 16846 consensus_queue.cc:260] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:30.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.639292 16856 sys_catalog.cc:455] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } } }
18:56:30.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.640702 16856 sys_catalog.cc:458] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [sys.catalog]: This master's current role is: FOLLOWER
18:56:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.645656 16846 sys_catalog.cc:564] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [sys.catalog]: configured and running, proceeding with master startup.
18:56:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:30.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
18:56:30.686 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:30.691 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc
18:56:30.706 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
18:56:30.708 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc
18:56:30.714 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
18:56:30.720 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
18:56:30.725 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
18:56:30.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
18:56:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.730214 16856 raft_consensus.cc:482] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:30.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.731236 16856 raft_consensus.cc:504] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.741585 16856 leader_election.cc:288] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049), 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721)
18:56:30.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:30.759349 16868 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:30.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:30.759856 16868 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:30.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:30.773645 16800 tablet.cc:2369] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:56:30.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:30 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:30.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
18:56:30.790 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:30.790 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc
18:56:30.802 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
18:56:30.803 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.48.189:36721
18:56:30.805 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
18:56:30.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.806510 16578 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16681
18:56:30.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.717587 16764 master_runner.cc:186] Error getting master registration for 127.16.48.190:40049: OK, instance_id { permanent_uuid: "7168256b087346a8ad0605d090e5142b" instance_seqno: 1781895384902885 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
18:56:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.847559 16764 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:30.851303 16797 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.16.48.189:36721 (error 108)
18:56:30.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:30.857 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.48.188:34995
18:56:30.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:30.857671 16797 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721): Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:30.858 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
18:56:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.867763 16578 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16764
18:56:30.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:30.914 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.48.190:40049
18:56:30.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
18:56:30.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.915050 16578 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16604
18:56:30.955 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:30.956 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.48.189:36721
18:56:30.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
18:56:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:30.960621 16578 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data
18:56:30.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/logs
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/info.pb
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.189:36721
18:56:30.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.189
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35177
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.189.keytab
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.189
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:56:31.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.347033 16873 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:31.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.347788 16873 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:31.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.348433 16873 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:31.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.386051 16873 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:56:31.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.386304 16873 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:31.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.386580 16873 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:31.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.386768 16873 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.386968 16873 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.387161 16873 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.387303 16873 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39769/jwks.json
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:31.428385 16873 master_runner.cc:386] Master server non-default flags:
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.189.keytab
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.189
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.189:36721
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/info.pb
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.189
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35177
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/logs
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:31.430117 16873 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:31.431247 16873 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.452138 16882 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.452280 16884 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:31.454849 16873 server_base.cc:1034] running on GCE node
18:56:32.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:31.453325 16881 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.708798 16873 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.712535 16873 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.714011 16873 hybrid_clock.cc:648] HybridClock initialized: now 1781895392713978 us; error 67 us; skew 500 ppm
18:56:32.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.714900 16873 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:32.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:32 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895392, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:32.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.721781 16873 init.cc:377] Logged in from keytab as kudu/127.16.48.189@KRBTEST.COM (short username kudu)
18:56:32.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.727528 16873 webserver.cc:460] Webserver started at http://127.16.48.189:35177/ using document root <none> and password file <none>
18:56:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.728601 16873 fs_manager.cc:359] Metadata directory not provided
18:56:32.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.728817 16873 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.738093 16873 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.000s
18:56:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.742998 16893 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.744309 16873 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
18:56:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.744688 16873 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "903421a259a74750a539cb85f24ef279"
18:56:32.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:27 on dist-test-slave-6bn0"
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.747520 16873 fs_report.cc:383] FS layout report
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/data
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:32.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:32.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:32.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.785653 16873 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.792583 16873 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:32.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.793097 16873 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:32.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.872793 16873 rpc_server.cc:307] RPC server started. Bound to: 127.16.48.189:36721
18:56:32.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.872924 16944 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.48.189:36721 every 8 connection(s)
18:56:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.875972 16873 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/data/info.pb
18:56:32.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.880198 16578 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16873
18:56:32.881 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:32.881 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.48.188:34995
18:56:32.881 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.883777 16578 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/logs
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/info.pb
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.188:34995
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.188
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44779
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.188.keytab
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.188
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:56:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.891187 16945 sys_catalog.cc:263] Verifying existing consensus state
18:56:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 18:56:32.892434 16945 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:56:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.892982 16873 master.cc:537] Master@127.16.48.189:36721 shutting down...
18:56:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:32.962206 16873 master.cc:552] Master@127.16.48.189:36721 shutdown complete.
18:56:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:56:33.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.251797 16947 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:33.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.252417 16947 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:33.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.253176 16947 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:33.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.286105 16947 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:56:33.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.286356 16947 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:33.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.286615 16947 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:33.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.286816 16947 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.287005 16947 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.287190 16947 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.287416 16947 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39769/jwks.json
18:56:33.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:33.326211 16947 master_runner.cc:386] Master server non-default flags:
18:56:33.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:33.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:33.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.188.keytab
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.188
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.188:34995
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/info.pb
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.188
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44779
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/logs
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:33.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:33.327848 16947 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:33.329077 16947 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:33.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.347929 16954 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:34.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.347985 16957 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:34.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:33.350723 16947 server_base.cc:1034] running on GCE node
18:56:34.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:33.348101 16955 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:34.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.611554 16947 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:34.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.614413 16947 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.615862 16947 hybrid_clock.cc:648] HybridClock initialized: now 1781895394615826 us; error 61 us; skew 500 ppm
18:56:34.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.616731 16947 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:34.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:34 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895394, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:34.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.623617 16947 init.cc:377] Logged in from keytab as kudu/127.16.48.188@KRBTEST.COM (short username kudu)
18:56:34.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.628816 16947 webserver.cc:460] Webserver started at http://127.16.48.188:44779/ using document root <none> and password file <none>
18:56:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.630118 16947 fs_manager.cc:359] Metadata directory not provided
18:56:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.630414 16947 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.643006 16947 fs_manager.cc:709] Time spent opening directory manager: real 0.008s user 0.005s sys 0.004s
18:56:34.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.648533 16966 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.649809 16947 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
18:56:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.650220 16947 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "743ebe9d3ff74b49a3f978aed51061ba"
18:56:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:29 on dist-test-slave-6bn0"
18:56:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.652993 16947 fs_report.cc:383] FS layout report
18:56:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/wal
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/data
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:34.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.711922 16947 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.718602 16947 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:34.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.719141 16947 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:34.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.795277 16947 rpc_server.cc:307] RPC server started. Bound to: 127.16.48.188:34995
18:56:34.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.795451 17017 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.48.188:34995 every 8 connection(s)
18:56:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.798875 16947 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-2/data/info.pb
18:56:34.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.802320 16578 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 16947
18:56:34.803 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:34.804 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.48.190:40049
18:56:34.804 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
18:56:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.805817 16578 external_mini_cluster.cc:1326] Running /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task01LnxB/build/tsan/bin/kudu
18:56:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:34.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/logs
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/info.pb
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
18:56:34.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.190:40049
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.190
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42119
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.190.keytab
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.190
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:34.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
18:56:34.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.811100 17018 sys_catalog.cc:263] Verifying existing consensus state
18:56:34.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.821995 17018 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Bootstrap starting.
18:56:34.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.857713 17018 log.cc:824] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Log is configured to *not* fsync() on all Append() calls
18:56:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.862686 17018 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
18:56:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.863554 17018 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Bootstrap complete.
18:56:34.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.889050 17018 raft_consensus.cc:348] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.891855 17018 raft_consensus.cc:374] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
18:56:34.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.892174 17018 raft_consensus.cc:729] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 743ebe9d3ff74b49a3f978aed51061ba, State: Initialized, Role: FOLLOWER
18:56:34.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.893225 17018 consensus_queue.cc:260] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:34.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.896318 17022 sys_catalog.cc:455] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } } }
18:56:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.897202 17022 sys_catalog.cc:458] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [sys.catalog]: This master's current role is: FOLLOWER
18:56:34.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.898110 17018 sys_catalog.cc:564] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [sys.catalog]: configured and running, proceeding with master startup.
18:56:34.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:34.931464 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:34.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:34.931829 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:34.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.933066 16947 master_runner.cc:186] Error getting master registration for 127.16.48.190:40049: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111),
18:56:34.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:34.955575 16947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:34.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:34.975019 16972 tablet.cc:2369] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: Can't schedule compaction. Clean time has not been advanced past its initial value.
18:56:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.179490 17020 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
18:56:35.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.180408 17020 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
18:56:35.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.181195 17020 flags.cc:425] Enabled unsafe flag: --never_fsync=true
18:56:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.215060 17020 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
18:56:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.215397 17020 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
18:56:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.215833 17020 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
18:56:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.216157 17020 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
18:56:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.216486 17020 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
18:56:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.216814 17020 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
18:56:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.217046 17020 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39769/jwks.json
18:56:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.261220 17020 master_runner.cc:386] Master server non-default flags:
18:56:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
18:56:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.48.148:35015
18:56:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
18:56:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
18:56:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data
18:56:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.48.190:40049,127.16.48.189:36721,127.16.48.188:34995
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/krb5kdc/kudu_127.16.48.190.keytab
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.48.190
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.48.190:40049
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39769/jwks.json
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/info.pb
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.48.190
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42119
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/logs
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
18:56:35.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
18:56:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
18:56:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
18:56:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
18:56:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 17:46:36 UTC on ccycloud.quasar-zpqdyv.root.comops.site
18:56:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4456156
18:56:35.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
18:56:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.264297 17020 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
18:56:35.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.265616 17020 file_cache.cc:493] Constructed file cache file cache with capacity 419430
18:56:35.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.288359 17042 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.348217 17022 raft_consensus.cc:482] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:35.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.349099 17022 raft_consensus.cc:504] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:35.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.357934 16968 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111)
18:56:35.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.367058 17022 leader_election.cc:288] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049), 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721)
18:56:35.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.388862 16968 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049): Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111)
18:56:35.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.393743 16969 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721): Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:35.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.394378 16969 leader_election.cc:302] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 743ebe9d3ff74b49a3f978aed51061ba; no voters: 7168256b087346a8ad0605d090e5142b, 903421a259a74750a539cb85f24ef279
18:56:35.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:35.396178 17022 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:56:35.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.938578 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:35.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.939814 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:36.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.291025 17041 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:35.288512 17044 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
18:56:36.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:36.869220 17043 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1579 milliseconds
18:56:36.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.869390 17020 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
18:56:36.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.871079 17020 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
18:56:36.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.874666 17020 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
18:56:36.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.876130 17020 hybrid_clock.cc:648] HybridClock initialized: now 1781895396876053 us; error 114 us; skew 500 ppm
18:56:36.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.877321 17020 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
18:56:36.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:36 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895396, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.885558 17020 init.cc:377] Logged in from keytab as kudu/127.16.48.190@KRBTEST.COM (short username kudu)
18:56:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.891901 17020 webserver.cc:460] Webserver started at http://127.16.48.190:42119/ using document root <none> and password file <none>
18:56:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.893304 17020 fs_manager.cc:359] Metadata directory not provided
18:56:36.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.893605 17020 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
18:56:36.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.905092 17020 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.008s sys 0.000s
18:56:36.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.911242 17055 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
18:56:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.912684 17020 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
18:56:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.912998 17020 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data,/tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7168256b087346a8ad0605d090e5142b"
18:56:36.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 18:56:24 on dist-test-slave-6bn0"
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.915467 17020 fs_report.cc:383] FS layout report
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/data
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
18:56:36.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
18:56:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:36.954195 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:36.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:36.954924 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:36.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.959259 17020 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
18:56:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.966202 17020 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
18:56:36.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:36.966653 17020 kserver.cc:163] Server-wide thread pool size limit: 3276
18:56:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.058955 17020 rpc_server.cc:307] RPC server started. Bound to: 127.16.48.190:40049
18:56:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.059075 17106 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.48.190:40049 every 8 connection(s)
18:56:37.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.062626 17020 server_base.cc:1166] Dumped server information to /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/data/info.pb
18:56:37.065 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
18:56:37.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.064060 16578 external_mini_cluster.cc:1388] Started /tmp/dist-test-task01LnxB/build/tsan/bin/kudu as pid 17020
18:56:37.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.083048 17107 sys_catalog.cc:263] Verifying existing consensus state
18:56:37.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 18:56:37.084609 17107 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:56:37.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.087739 17020 master.cc:537] Master@127.16.48.190:40049 shutting down...
18:56:37.154 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.226524 17020 master.cc:552] Master@127.16.48.190:40049 shutdown complete.
18:56:37.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task01LnxB/test-tmp/mini-kudu-cluster7909184776044275126/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
18:56:37.249 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.48.188:34995(127.16.48.188:34995)] connecting to peer
18:56:37.259 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.326 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.327 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.327 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.16.48.188:34995(127.16.48.188:34995)] Successfully connected to peer
18:56:37.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.331 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.430644 17113 raft_consensus.cc:482] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.431576 17113 raft_consensus.cc:504] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:37.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.434520 17113 leader_election.cc:288] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049), 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721)
18:56:37.434 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
18:56:37.455 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.16.48.188
18:56:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:37.478860 16968 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049): Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111)
18:56:37.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:37.479522 16969 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721): Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:37.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.479938 16969 leader_election.cc:302] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 743ebe9d3ff74b49a3f978aed51061ba; no voters: 7168256b087346a8ad0605d090e5142b, 903421a259a74750a539cb85f24ef279
18:56:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:37.480703 17113 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:56:37.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:37 dist-test-slave-6bn0 krb5kdc[16591](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895390, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.48.188@KRBTEST.COM
18:56:37.733 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xff44a32c, L:/127.0.0.1:47436 - R:/127.16.48.188:34995] using SASL/GSSAPI
18:56:37.754 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:37.773 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.779 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.780 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.782 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.783 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.785 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:37.792 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.793 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.796 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.796 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.797 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.800 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.800 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.801 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.803 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:37.812 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.814 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.817 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.817 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.821 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.822 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.824 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:37.833 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.836 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.842 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.842 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.847 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.851 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:37.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.898 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.902 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:37.912 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.915 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.917 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.918 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.920 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.921 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.921 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.922 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:37.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:37.958011 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:37.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:37.958300 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:37.993 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:37.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:37.997 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.998 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.998 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:37.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:37.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.001 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:38.053 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:38.055 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:38.057 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.057 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.057 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.059 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.061 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:38.373 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:38.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:38.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.378 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.378 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.379 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.380 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:38.892 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:38.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:38.896 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.896 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.896 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.896 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.896 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:38.896 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:38.899 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:38.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:38.961293 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:38.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:38.961565 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:39.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:39.958401 16947 master_runner.cc:186] Error getting master registration for 127.16.48.190:40049: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111),
18:56:39.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:39.958783 16947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:39.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:39.964491 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:39.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:39.964745 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:40.156905 17120 raft_consensus.cc:482] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:40.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:40.157313 17120 raft_consensus.cc:504] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:40.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:40.159011 17120 leader_election.cc:288] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049), 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721)
18:56:40.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:40.163679 16968 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049): Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111)
18:56:40.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:40.164109 16969 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721): Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:40.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:40.164398 16969 leader_election.cc:302] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 743ebe9d3ff74b49a3f978aed51061ba; no voters: 7168256b087346a8ad0605d090e5142b, 903421a259a74750a539cb85f24ef279
18:56:40.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:40.164922 17120 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:56:40.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:40.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:40.376 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:40.376 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:40.377 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:40.377 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:40.377 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:40.377 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:40.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:40.967574 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:40.967911 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:41.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:41.970815 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:41.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:41.971140 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:42.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:42.367877 16423 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb0c0 after lost signal to thread 16297
18:56:42.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:42.368832 16423 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbd20 after lost signal to thread 16426
18:56:42.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:42.974510 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:42.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:42.974819 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:43.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:43.974 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:43.976 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:43.976 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:43.976 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:43.980 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:43.980 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:43.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:43.981 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:43.980868 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:43.981334 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:44 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895404, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:44.627782 16962 init.cc:295] Successfully reacquired a new kerberos TGT
18:56:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:44.962581 16947 master_runner.cc:186] Error getting master registration for 127.16.48.190:40049: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111),
18:56:44.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:44.963201 16947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:44.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:44.984844 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:44.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:44.985133 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:45.174789 17129 raft_consensus.cc:482] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:45.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:45.175302 17129 raft_consensus.cc:504] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:45.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:45.177026 17129 leader_election.cc:288] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049), 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721)
18:56:45.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:45.178360 16968 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111) [suppressed 5 similar messages]
18:56:45.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:45.181181 16968 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049): Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111)
18:56:45.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:45.181699 16969 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721): Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:45.181993 16969 leader_election.cc:302] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 743ebe9d3ff74b49a3f978aed51061ba; no voters: 7168256b087346a8ad0605d090e5142b, 903421a259a74750a539cb85f24ef279
18:56:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:45.182597 17129 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:56:45.412 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:45.414 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:45.416 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:45.416 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:45.416 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:45.416 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:45.417 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:45.417 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:45.419 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:45.988049 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:45.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:45.988354 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:46.572 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:46.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:46.577 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:46.577 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:46.577 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:46.578 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:46.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:46.578 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:46.580 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:46.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:46.991362 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:46.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:46.991657 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:48.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:48.005625 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:48.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:48.006949 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:48.170498 17136 raft_consensus.cc:482] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:48.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:48.173785 17136 raft_consensus.cc:504] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:48.185185 17136 leader_election.cc:288] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049), 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721)
18:56:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:48.209918 16968 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049): Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111)
18:56:48.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:48.221030 16969 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721): Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:48.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:48.223847 16969 leader_election.cc:302] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 743ebe9d3ff74b49a3f978aed51061ba; no voters: 7168256b087346a8ad0605d090e5142b, 903421a259a74750a539cb85f24ef279
18:56:48.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:48.227854 17136 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:56:48.435 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:48.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:48.462 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:48.465 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:48.466 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:48.484 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:48.485 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:48.486 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:48.497 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:49.024446 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:49.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:49.026199 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:49.966902 16947 master_runner.cc:186] Error getting master registration for 127.16.48.190:40049: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111),
18:56:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:49.967422 16947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:50.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:50.032253 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:50.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:50.032596 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:50.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:50.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:50.259 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:50.259 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:50.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:50.259 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:50.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:50.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:50.261 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:50.773 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:50.775 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:50.777 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:50.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:50.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:50.777 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:50.777 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:50.777 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:50.781 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:50.806032 17144 raft_consensus.cc:482] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
18:56:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:50.806548 17144 raft_consensus.cc:504] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7168256b087346a8ad0605d090e5142b" member_type: VOTER last_known_addr { host: "127.16.48.190" port: 40049 } } peers { permanent_uuid: "903421a259a74750a539cb85f24ef279" member_type: VOTER last_known_addr { host: "127.16.48.189" port: 36721 } } peers { permanent_uuid: "743ebe9d3ff74b49a3f978aed51061ba" member_type: VOTER last_known_addr { host: "127.16.48.188" port: 34995 } }
18:56:50.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:50.808796 17144 leader_election.cc:288] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049), 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721)
18:56:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:50.811632 16968 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111) [suppressed 3 similar messages]
18:56:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:50.816490 16968 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7168256b087346a8ad0605d090e5142b (127.16.48.190:40049): Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111)
18:56:50.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:50.818331 16969 leader_election.cc:334] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 903421a259a74750a539cb85f24ef279 (127.16.48.189:36721): Network error: Client connection negotiation failed: client connection to 127.16.48.189:36721: connect: Connection refused (error 111)
18:56:50.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:50.818730 16969 leader_election.cc:302] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 743ebe9d3ff74b49a3f978aed51061ba; no voters: 7168256b087346a8ad0605d090e5142b, 903421a259a74750a539cb85f24ef279
18:56:50.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:50.819643 17144 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
18:56:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:51.036279 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:51.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:51.036659 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:52.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:52.040050 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:52.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:52.040395 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:53.043762 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:53.044106 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:54.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:54.048554 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:54.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:54.048975 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 18:56:54 dist-test-slave-6bn0 krb5kdc[16591](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781895414, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.48.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
18:56:54.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:54.631510 16962 init.cc:295] Successfully reacquired a new kerberos TGT
18:56:54.712 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:54.714 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:54.716 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:54.716 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:54.716 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:54.716 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:54.716 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:54.716 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:54.719 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:54.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:54.895 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:54.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:54.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:54.897 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:54.899 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:54.899 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:54.899 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:54.901 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:54.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:54.971271 16947 master_runner.cc:186] Error getting master registration for 127.16.48.190:40049: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111),
18:56:54.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:54.971802 16947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:56:55.013 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:55.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:55.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:55.016 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:55.016 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:55.019 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:55.019 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:55.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:55.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:55.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:55.052299 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:55.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:55.052620 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:56.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:56.056131 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:56.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:56.056483 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:56.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:56.157123 16563 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd000 after lost signal to thread 16438
18:56:56.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:56.157989 16563 debug-util.cc:398] Leaking SignalData structure 0x7b08000af320 after lost signal to thread 16566
18:56:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:57.061602 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:57.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:57.062049 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:58.065213 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:58.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:58.065507 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:58.153 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:56:58.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:56:58.155 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:58.155 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:58.155 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:58.155 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:58.156 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:56:58.156 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:56:58.158 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:56:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:59.068493 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:56:59.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:56:59.068768 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:56:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:59.974998 16947 master_runner.cc:186] Error getting master registration for 127.16.48.190:40049: Network error: Client connection negotiation failed: client connection to 127.16.48.190:40049: connect: Connection refused (error 111),
18:56:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:56:59.975512 16947 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
18:57:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:00.072479 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:57:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:00.072857 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:57:00.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:57:00.773 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:57:00.774 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:00.774 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:00.775 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:00.775 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:00.775 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:00.775 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:00.777 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:57:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:01.077003 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:57:01.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:01.077391 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:57:01.112 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:57:01.113 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:57:01.114 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:01.115 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:01.115 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:01.117 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:01.117 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:01.117 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:01.118 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:57:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:02.081836 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:57:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:02.082221 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:57:03.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:03.086326 17033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 743ebe9d3ff74b49a3f978aed51061ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
18:57:03.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 18:57:03.086614 17033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
18:57:03.553 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.189:36721(127.16.48.189:36721)] connecting to peer
18:57:03.554 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.16.48.190:40049(127.16.48.190:40049)] connecting to peer
18:57:03.555 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:03.555 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:03.555 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.48.190:40049(127.16.48.190:40049)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:03.556 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.189:36721(127.16.48.189:36721)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:03.556 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.190:40049: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049
18:57:03.556 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.48.189:36721: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
18:57:03.560 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049; will retry
18:57:03.564 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26491), Traces: [0ms] refreshing cache from master, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [239ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [660ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [689ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [693ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [696ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [697ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [702ms] refreshing cache from master, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [707ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [711ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [713ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [728ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [732ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [734ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [736ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [746ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [747ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [753ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [759ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [762ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [803ms] refreshing cache from master, [803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [808ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [810ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [813ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [814ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [823ms] refreshing cache from master, [823ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [831ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [832ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [834ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [903ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [909ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [910ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [911ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [912ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [963ms] refreshing cache from master, [964ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [965ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [966ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [971ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [972ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1283ms] refreshing cache from master, [1284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1286ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1291ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1292ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1803ms] refreshing cache from master, [1803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1809ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1810ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3283ms] refreshing cache from master, [3283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [3284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [3285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [3290ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [3291ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [6883ms] refreshing cache from master, [6883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [6887ms] too many traces: truncated at 100 traces, deferred=Deferred@1742704441(state=PENDING, result=null, callback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884), errback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884)))
18:57:03.570 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.48.189:36721(127.16.48.189:36721)] disconnecting while in state TERMINATED
18:57:03.571 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.48.188:34995(127.16.48.188:34995)] disconnecting while in state READY
18:57:03.571 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.16.48.190:40049(127.16.48.190:40049)] disconnecting while in state TERMINATED
18:57:03.578 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:57:03.580 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:57:03.582 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.16.48.188:34995(127.16.48.188:34995)] handling channelInactive
18:57:03.582 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.48.188:34995(127.16.48.188:34995)] cleaning up while in state READY due to: connection closed
18:57:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:57:03.590194 16578 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16947
18:57:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:57:04Z chronyd exiting
18:57:05.852 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
18:57:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:57:05.854717 15847 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16146
18:57:05.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:57:05.891211 15847 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16291
18:57:05.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:57:05.931115 15847 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16437
18:57:05.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:57:05.964702 15847 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 15859
18:57:06.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:57:06.001447 15847 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 15929
18:57:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 18:57:06.043982 15847 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task01LnxB/build/tsan/bin/kudu with pid 16004
18:57:07.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T18:57:07Z chronyd exiting
18:57:08.324 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26491), Traces: [0ms] refreshing cache from master, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [239ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [660ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [689ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [693ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [696ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [697ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [702ms] refreshing cache from master, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [707ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [711ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [713ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [728ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [732ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [734ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [736ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [746ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [747ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [753ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [759ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [762ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [803ms] refreshing cache from master, [803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [808ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [810ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [813ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [814ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [823ms] refreshing cache from master, [823ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [831ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [832ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [834ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [903ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [909ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [910ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [911ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [912ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [963ms] refreshing cache from master, [964ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [965ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [966ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [971ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [972ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1283ms] refreshing cache from master, [1284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1286ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1291ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1292ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1803ms] refreshing cache from master, [1803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1809ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1810ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3283ms] refreshing cache from master, [3283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [3284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [3285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [3290ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [3291ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [6883ms] refreshing cache from master, [6883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [6887ms] too many traces: truncated at 100 traces, deferred=Deferred@1742704441(state=PENDING, result=null, callback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884), errback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.48.189:36721
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26491), Traces: [0ms] refreshing cache from master, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [239ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [660ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [689ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [693ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [696ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [697ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [702ms] refreshing cache from master, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [707ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [711ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [713ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [728ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [732ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [734ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [736ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [746ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [747ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [753ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [759ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [762ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [803ms] refreshing cache from master, [803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [808ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [810ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [813ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [814ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [823ms] refreshing cache from master, [823ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [831ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [832ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [834ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [903ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [909ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [910ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [911ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [912ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [963ms] refreshing cache from master, [964ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [965ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [966ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [971ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [972ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1283ms] refreshing cache from master, [1284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1286ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1291ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1292ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1803ms] refreshing cache from master, [1803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1809ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1810ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3283ms] refreshing cache from master, [3283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [3284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [3285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [3290ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [3291ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [6883ms] refreshing cache from master, [6883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [6887ms] too many traces: truncated at 100 traces, deferred=Deferred@1742704441(state=PENDING, result=null, callback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884), errback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.48.189:36721
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
18:57:08.849 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
18:57:08.849 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26491), Traces: [0ms] refreshing cache from master, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [239ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [660ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [689ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [693ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [696ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [697ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [702ms] refreshing cache from master, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [707ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [711ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [713ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [728ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [732ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [734ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [736ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [746ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [747ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [753ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [759ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [762ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [803ms] refreshing cache from master, [803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [808ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [810ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [813ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [814ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [823ms] refreshing cache from master, [823ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [831ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [832ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [834ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [903ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [909ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [910ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [911ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [912ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [963ms] refreshing cache from master, [964ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [965ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [966ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [971ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [972ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1283ms] refreshing cache from master, [1284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1286ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1291ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1292ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1803ms] refreshing cache from master, [1803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1809ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1810ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3283ms] refreshing cache from master, [3283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [3284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [3285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [3290ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [3291ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [6883ms] refreshing cache from master, [6883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [6887ms] too many traces: truncated at 100 traces, deferred=Deferred@1742704441(state=PENDING, result=null, callback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884), errback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.48.189:36721
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=26491), Traces: [0ms] refreshing cache from master, [53ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [239ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [660ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [683ms] refreshing cache from master, [684ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [686ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [689ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [693ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [696ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [697ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [702ms] refreshing cache from master, [703ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [704ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [707ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [711ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [713ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [714ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [723ms] refreshing cache from master, [723ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [724ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [725ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [728ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [732ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [734ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [736ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [743ms] refreshing cache from master, [743ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [746ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [747ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [753ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [759ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [761ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [762ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [803ms] refreshing cache from master, [803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [808ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [810ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [813ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [814ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [823ms] refreshing cache from master, [823ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [824ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [826ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [829ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [831ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [832ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [834ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [903ms] refreshing cache from master, [904ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [906ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [909ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [910ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [911ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [912ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [963ms] refreshing cache from master, [964ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [965ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [966ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [968ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [971ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [972ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1283ms] refreshing cache from master, [1284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1286ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1291ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1292ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1803ms] refreshing cache from master, [1803ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [1804ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [1805ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [1807ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [1809ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [1810ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3283ms] refreshing cache from master, [3283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [3284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [3285ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.190:40049: Network error: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049, [3288ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.189:36721: Network error: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [3290ms] Sub RPC ConnectToMaster: received response from server master-127.16.48.188:34995: OK, [3291ms] delaying RPC due to: Service unavailable: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721, [6883ms] refreshing cache from master, [6883ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.189:36721, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.188:34995, [6885ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.48.190:40049, [6887ms] too many traces: truncated at 100 traces, deferred=Deferred@1742704441(state=PENDING, result=null, callback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884), errback=(continuation of Deferred@451349858 after retry RPC after error@179571385) -> (continuation of Deferred@1045932765 after retry RPC after error@636111977) -> (continuation of Deferred@121952683 after retry RPC after error@764410424) -> (continuation of Deferred@772652054 after retry RPC after error@1509106874) -> (continuation of Deferred@2019243950 after retry RPC after error@710364327) -> (continuation of Deferred@1769584887 after retry RPC after error@703567766) -> (continuation of Deferred@136852290 after retry RPC after error@403338388) -> (continuation of Deferred@1007733090 after retry RPC after error@1763093073) -> (continuation of Deferred@552412690 after retry RPC after error@1795704911) -> (continuation of Deferred@1365890965 after retry RPC after error@225553355) -> (continuation of Deferred@1147523521 after retry RPC after error@1837540146) -> (continuation of Deferred@1925977192 after retry RPC after error@1927529993) -> (continuation of Deferred@409139789 after retry RPC after error@1076188480) -> (continuation of Deferred@1381529208 after retry RPC after error@1055338932) -> (continuation of Deferred@827851959 after retry RPC after error@1530775689) -> (continuation of Deferred@1254596128 after retry RPC after error@1294625885) -> (continuation of Deferred@1009169651 after retry RPC after error@166353534) -> (continuation of Deferred@2141910013 after retry RPC after error@1635528913) -> (continuation of Deferred@1752465697 after retry RPC after error@857087995) -> (continuation of Deferred@453719935 after retry RPC after error@468809335) -> (continuation of Deferred@1463845281 after retry RPC after error@1412085109) -> (continuation of Deferred@383303744 after retry RPC after error@101338658) -> (continuation of Deferred@1435802315 after retry RPC after error@1410599953) -> (continuation of Deferred@69082131 after retry RPC after error@1353787884)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.48.189:36721,127.16.48.188:34995,127.16.48.190:40049) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.190:40049(127.16.48.190:40049): Connection refused: /127.16.48.190:40049,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.48.189:36721(127.16.48.189:36721): Connection refused: /127.16.48.189:36721
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.48.189:36721
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)