03:12:18.831 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:12:18.843 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskXqVMDO/test-tmp
03:12:18.845 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:12:18.849 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskXqVMDO/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:12:19.062 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:12:19.245 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:12:19.246 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:12:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.251432 3781 mini_oidc.cc:152] Starting JWKS server
03:12:19.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.255887 3781 webserver.cc:460] Webserver started at http://127.0.0.1:34161/ using document root <none> and password file <none>
03:12:19.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.256506 3781 mini_oidc.cc:189] Starting OIDC Discovery server
03:12:19.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.259007 3781 webserver.cc:460] Webserver started at http://127.0.0.1:46535/ using document root <none> and password file <none>
03:12:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:12:19Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:12:19.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:12:19Z Disabled control of system clock
03:12:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.308241 3781 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/wal
03:12:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data
03:12:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:19.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/logs
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data/info.pb
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.126:40713
03:12:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.126
03:12:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:19.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:19.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34161/jwks.json with env {}
03:12:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.613093 3793 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.613691 3793 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.614362 3793 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:19.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.654035 3793 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.655296 3793 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.655547 3793 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.655817 3793 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.656044 3793 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.656244 3793 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34161/jwks.json
03:12:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.690945 3793 master_runner.cc:386] Master server non-default flags:
03:12:19.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:19.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:19.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:19.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:19.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data
03:12:19.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/wal
03:12:19.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:19.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:19.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:19.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.126:40713
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34161/jwks.json
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data/info.pb
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.126
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/logs
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.692234 3793 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:19.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.693940 3793 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:19.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.712270 3799 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:20.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.712294 3802 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:20.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:19.713481 3800 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:20.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:19.713564 3793 server_base.cc:1034] running on GCE node
03:12:20.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.904639 3793 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:20.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.907835 3793 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:20.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.909349 3793 hybrid_clock.cc:648] HybridClock initialized: now 1790219540909270 us; error 81 us; skew 500 ppm
03:12:20.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.910624 3793 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:20.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.917404 3793 webserver.cc:460] Webserver started at http://127.3.177.126:32963/ using document root <none> and password file <none>
03:12:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.918479 3793 fs_manager.cc:359] Metadata directory not provided
03:12:20.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.918736 3793 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:20.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.919270 3793 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:20.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.924363 3793 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data/instance:
03:12:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15177b238f2e4b528936b63a613d85e1"
03:12:20.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:20 on dist-test-slave-lltt"
03:12:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.925823 3793 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/wal/instance:
03:12:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15177b238f2e4b528936b63a613d85e1"
03:12:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:20 on dist-test-slave-lltt"
03:12:20.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.933784 3793 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
03:12:20.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.940114 3809 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:20.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.941463 3793 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:12:20.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.941869 3793 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/wal
03:12:20.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15177b238f2e4b528936b63a613d85e1"
03:12:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:20 on dist-test-slave-lltt"
03:12:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.942313 3793 fs_report.cc:383] FS layout report
03:12:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/wal
03:12:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/wal
03:12:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data/data
03:12:20.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:20.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:20.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:20.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.979487 3793 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:20.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.981256 3793 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:20.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:20.981868 3793 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.057075 3793 rpc_server.cc:307] RPC server started. Bound to: 127.3.177.126:40713
03:12:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.057224 3860 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.177.126:40713 every 8 connection(s)
03:12:21.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.059927 3793 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/data/info.pb
03:12:21.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.064428 3781 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 3793
03:12:21.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.064863 3781 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-0/wal/instance
03:12:21.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.066810 3861 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:12:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.068607 3781 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/wal
03:12:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data
03:12:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:21.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/logs
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data/info.pb
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:21.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.125:39077
03:12:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.125
03:12:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34161/jwks.json with env {}
03:12:21.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.081271 3861 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } has no permanent_uuid. Determining permanent_uuid...
03:12:21.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.110445 3861 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } has no permanent_uuid. Determining permanent_uuid...
03:12:21.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.112610 3810 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:21.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.117117 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.125:39077: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.166716 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } attempt: 1
03:12:21.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.170730 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.125:39077: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.239107 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } attempt: 2
03:12:21.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.243323 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.125:39077: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:21.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.334755 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } attempt: 3
03:12:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.338541 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.125:39077: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:21.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.389676 3863 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:21.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.390282 3863 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:21.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.390851 3863 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.421811 3863 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.422113 3863 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.422312 3863 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.422490 3863 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.422709 3863 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:21.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.422950 3863 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34161/jwks.json
03:12:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.458882 3863 master_runner.cc:386] Master server non-default flags:
03:12:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:21.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/wal
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.125:39077
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:21.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34161/jwks.json
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data/info.pb
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.125
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/logs
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:21.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:21.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:21.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:21.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:21.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:21.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.460116 3863 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:21.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.462219 3863 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:21.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.482168 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } attempt: 4
03:12:21.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.483172 3875 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.486420 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.125:39077: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.786170 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } attempt: 5
03:12:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.791028 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.125:39077: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:22.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.338429 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } attempt: 6
03:12:22.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:22.347292 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.125:39077: Network error: Client connection negotiation failed: client connection to 127.3.177.125:39077: connect: Connection refused (error 111)
03:12:22.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.483856 3872 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:22.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:21.484453 3873 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:22.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:21.487409 3863 server_base.cc:1034] running on GCE node
03:12:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.686818 3863 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:22.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.690021 3863 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:22.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.691406 3863 hybrid_clock.cc:648] HybridClock initialized: now 1790219542691375 us; error 56 us; skew 500 ppm
03:12:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.692456 3863 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:22.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.705685 3863 webserver.cc:460] Webserver started at http://127.3.177.125:34321/ using document root <none> and password file <none>
03:12:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.706673 3863 fs_manager.cc:359] Metadata directory not provided
03:12:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.706866 3863 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:22.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.707232 3863 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.712028 3863 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data/instance:
03:12:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3143ff6434334d6cab70d9b84c7e07eb"
03:12:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:22 on dist-test-slave-lltt"
03:12:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.713164 3863 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/wal/instance:
03:12:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3143ff6434334d6cab70d9b84c7e07eb"
03:12:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:22 on dist-test-slave-lltt"
03:12:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.720141 3863 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
03:12:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.725880 3884 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.726948 3863 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:12:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.727308 3863 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/wal
03:12:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3143ff6434334d6cab70d9b84c7e07eb"
03:12:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:22 on dist-test-slave-lltt"
03:12:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.727708 3863 fs_report.cc:383] FS layout report
03:12:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/wal
03:12:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/wal
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data/data
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.765017 3863 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.767035 3863 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.767575 3863 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:22.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.844901 3863 rpc_server.cc:307] RPC server started. Bound to: 127.3.177.125:39077
03:12:22.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.844991 3935 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.177.125:39077 every 8 connection(s)
03:12:22.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.847735 3863 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/data/info.pb
03:12:22.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.853060 3936 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:12:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.853852 3781 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 3863
03:12:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.854317 3781 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-1/wal/instance
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.857722 3781 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/wal
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/logs
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data/info.pb
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.124:45585
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.124
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:22.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34161/jwks.json with env {}
03:12:22.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.868527 3936 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } has no permanent_uuid. Determining permanent_uuid...
03:12:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.897303 3936 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } has no permanent_uuid. Determining permanent_uuid...
03:12:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.913650 3936 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } has no permanent_uuid. Determining permanent_uuid...
03:12:22.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:22.917045 3886 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:22.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:22.921028 3936 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:22.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:22.970598 3936 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 1
03:12:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:22.974323 3936 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.042701 3936 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 2
03:12:23.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.046416 3936 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.137795 3936 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 3
03:12:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.141880 3936 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.185415 3938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.185967 3938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.186507 3938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.217696 3938 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.217978 3938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.218174 3938 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.218387 3938 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.218622 3938 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.218770 3938 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34161/jwks.json
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.253806 3938 master_runner.cc:386] Master server non-default flags:
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/wal
03:12:23.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.124:45585
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34161/jwks.json
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data/info.pb
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.124
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/logs
03:12:23.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:23.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.255100 3938 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:23.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.257346 3938 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.280241 3951 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.285301 3936 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 4
03:12:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.289016 3936 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.407747 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } attempt: 7
03:12:23.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.426231 3861 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } has no permanent_uuid. Determining permanent_uuid...
03:12:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.432621 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.491025 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 1
03:12:23.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.495044 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.576640 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 2
03:12:23.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.581053 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.588429 3936 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 5
03:12:23.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.592722 3936 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.666674 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 3
03:12:23.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.670642 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.811780 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 4
03:12:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.815871 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.099457 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 5
03:12:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:24.103413 3861 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:24.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.140106 3936 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 6
03:12:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:24.147881 3936 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.177.124:45585: Network error: Client connection negotiation failed: client connection to 127.3.177.124:45585: connect: Connection refused (error 111)
03:12:24.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.280350 3949 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:24.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:23.282927 3938 server_base.cc:1034] running on GCE node
03:12:24.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:23.281970 3948 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.493155 3938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.496236 3938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.497654 3938 hybrid_clock.cc:648] HybridClock initialized: now 1790219544497618 us; error 68 us; skew 500 ppm
03:12:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.498467 3938 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.505301 3938 webserver.cc:460] Webserver started at http://127.3.177.124:45497/ using document root <none> and password file <none>
03:12:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.506181 3938 fs_manager.cc:359] Metadata directory not provided
03:12:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.506358 3938 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.506749 3938 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.512022 3938 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data/instance:
03:12:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aadaa136f8e64c899b8b06f4293db1f0"
03:12:24.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:24 on dist-test-slave-lltt"
03:12:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.513087 3938 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/wal/instance:
03:12:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aadaa136f8e64c899b8b06f4293db1f0"
03:12:24.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:24 on dist-test-slave-lltt"
03:12:24.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.520902 3938 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
03:12:24.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.527433 3962 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:24.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.528622 3938 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:12:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.528908 3938 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/wal
03:12:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aadaa136f8e64c899b8b06f4293db1f0"
03:12:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:24 on dist-test-slave-lltt"
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.529379 3938 fs_report.cc:383] FS layout report
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/wal
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/wal
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data/data
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:24.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.576386 3938 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:24.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.578235 3938 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:24.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.578650 3938 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:24.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.655997 3861 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 6
03:12:24.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.658730 3938 rpc_server.cc:307] RPC server started. Bound to: 127.3.177.124:45585
03:12:24.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.658808 4013 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.177.124:45585 every 8 connection(s)
03:12:24.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.662828 3938 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/data/info.pb
03:12:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.666673 3781 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 3938
03:12:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.667217 3781 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/master-2/wal/instance
03:12:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.669391 4016 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:12:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.691851 4016 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } has no permanent_uuid. Determining permanent_uuid...
03:12:24.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.724891 3861 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1: Bootstrap starting.
03:12:24.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.730270 4016 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } has no permanent_uuid. Determining permanent_uuid...
03:12:24.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.732689 3861 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1: Neither blocks nor log segments found. Creating new log.
03:12:24.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.735412 3861 log.cc:824] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1: Log is configured to *not* fsync() on all Append() calls
03:12:24.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.745877 4016 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } has no permanent_uuid. Determining permanent_uuid...
03:12:24.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.746485 3861 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1: No bootstrap required, opened a new log
03:12:24.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.772993 3861 raft_consensus.cc:348] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.773852 3861 raft_consensus.cc:374] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:12:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.774151 3861 raft_consensus.cc:729] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 15177b238f2e4b528936b63a613d85e1, State: Initialized, Role: FOLLOWER
03:12:24.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.775331 3861 consensus_queue.cc:260] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:24.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.778359 4016 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: Bootstrap starting.
03:12:24.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.779057 4025 sys_catalog.cc:455] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:24.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.779911 4025 sys_catalog.cc:458] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: This master's current role is: FOLLOWER
03:12:24.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.780490 3861 sys_catalog.cc:564] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: configured and running, proceeding with master startup.
03:12:24.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.786211 4016 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: Neither blocks nor log segments found. Creating new log.
03:12:24.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.788842 4016 log.cc:824] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: Log is configured to *not* fsync() on all Append() calls
03:12:24.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.794463 4016 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: No bootstrap required, opened a new log
03:12:24.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:24.818425 4037 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:24.820916 4037 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:24.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.821146 4025 raft_consensus.cc:482] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:12:24.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.821700 4025 raft_consensus.cc:504] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:24.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.824182 4025 leader_election.cc:288] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3143ff6434334d6cab70d9b84c7e07eb (127.3.177.125:39077), aadaa136f8e64c899b8b06f4293db1f0 (127.3.177.124:45585)
03:12:24.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.827576 4016 raft_consensus.cc:348] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:24.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.828604 4016 raft_consensus.cc:374] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:12:24.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.828904 4016 raft_consensus.cc:729] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aadaa136f8e64c899b8b06f4293db1f0, State: Initialized, Role: FOLLOWER
03:12:24.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.830070 4016 consensus_queue.cc:260] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.813941 3793 master_runner.cc:186] Error getting master registration for 127.3.177.125:39077: OK, instance_id { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" instance_seqno: 1790219542824240 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:12:24.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.854609 3793 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:12:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.861449 4039 sys_catalog.cc:455] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.862519 4039 sys_catalog.cc:458] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [sys.catalog]: This master's current role is: FOLLOWER
03:12:24.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.863894 4016 sys_catalog.cc:564] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [sys.catalog]: configured and running, proceeding with master startup.
03:12:24.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.864508 3989 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "15177b238f2e4b528936b63a613d85e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aadaa136f8e64c899b8b06f4293db1f0" is_pre_election: true
03:12:24.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.865504 3989 raft_consensus.cc:2454] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 15177b238f2e4b528936b63a613d85e1 in term 0.
03:12:24.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.867370 3811 leader_election.cc:302] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 15177b238f2e4b528936b63a613d85e1, aadaa136f8e64c899b8b06f4293db1f0; no voters:
03:12:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.868286 4025 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:12:24.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.868613 4025 raft_consensus.cc:482] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:12:24.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.868916 4025 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 0 FOLLOWER]: Advancing to term 1
03:12:24.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.876417 4025 raft_consensus.cc:504] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:24.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.880900 4025 leader_election.cc:288] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [CANDIDATE]: Term 1 election: Requested vote from peers 3143ff6434334d6cab70d9b84c7e07eb (127.3.177.125:39077), aadaa136f8e64c899b8b06f4293db1f0 (127.3.177.124:45585)
03:12:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.882586 3989 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "15177b238f2e4b528936b63a613d85e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "aadaa136f8e64c899b8b06f4293db1f0"
03:12:24.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.883147 3989 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [term 0 FOLLOWER]: Advancing to term 1
03:12:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.905242 3989 raft_consensus.cc:2454] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 15177b238f2e4b528936b63a613d85e1 in term 1.
03:12:24.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.906772 3811 leader_election.cc:302] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 15177b238f2e4b528936b63a613d85e1, aadaa136f8e64c899b8b06f4293db1f0; no voters:
03:12:24.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.907610 4025 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 1 FOLLOWER]: Leader election won for term 1
03:12:24.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.912930 4025 raft_consensus.cc:686] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [term 1 LEADER]: Becoming Leader. State: Replica: 15177b238f2e4b528936b63a613d85e1, State: Running, Role: LEADER
03:12:24.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.914211 4025 consensus_queue.cc:237] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.933725 4045 sys_catalog.cc:455] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 15177b238f2e4b528936b63a613d85e1. Latest consensus state: current_term: 1 leader_uuid: "15177b238f2e4b528936b63a613d85e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:24.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.934612 4045 sys_catalog.cc:458] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: This master's current role is: LEADER
03:12:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.942070 4050 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:12:24.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.952236 4050 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:12:24.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.976163 3989 raft_consensus.cc:1261] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [term 1 FOLLOWER]: Refusing update from remote peer 15177b238f2e4b528936b63a613d85e1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:12:24.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.979468 4045 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:12:24.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:24.991044 3815 tablet.cc:2369] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:12:25.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:25.005789 4053 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:25.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:25.006150 4053 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:24.958231 3938 master_runner.cc:186] Error getting master registration for 127.3.177.125:39077: OK, instance_id { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" instance_seqno: 1790219542824240 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:12:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.030473 3938 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:12:25.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.052227 4039 sys_catalog.cc:455] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 15177b238f2e4b528936b63a613d85e1. Latest consensus state: current_term: 1 leader_uuid: "15177b238f2e4b528936b63a613d85e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:25.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.052836 4039 sys_catalog.cc:458] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [sys.catalog]: This master's current role is: FOLLOWER
03:12:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.061076 4055 mvcc.cc:205] Tried to move back new op lower bound from 7332739256204939264 to 7332739255982297088. Current Snapshot: MvccSnapshot[applied={T|T < 7332739256204939264}]
03:12:25.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.068235 4056 mvcc.cc:205] Tried to move back new op lower bound from 7332739256204939264 to 7332739255982297088. Current Snapshot: MvccSnapshot[applied={T|T < 7332739256204939264}]
03:12:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.066519 4025 sys_catalog.cc:455] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "15177b238f2e4b528936b63a613d85e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.073743 4025 sys_catalog.cc:458] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: This master's current role is: LEADER
03:12:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.075565 4050 catalog_manager.cc:1347] Generated new cluster ID: e60dc3f85f76445d9292ae46a6f24a2a
03:12:25.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.075871 4050 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:12:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.094506 4039 sys_catalog.cc:455] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "15177b238f2e4b528936b63a613d85e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.095330 4039 sys_catalog.cc:458] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0 [sys.catalog]: This master's current role is: FOLLOWER
03:12:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.152313 4050 catalog_manager.cc:1370] Generated new certificate authority record
03:12:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.155385 4050 catalog_manager.cc:1504] Loading token signing keys...
03:12:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.182246 4050 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1: Generated new TSK 0
03:12:25.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.184685 4050 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:12:25.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.209369 3936 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } attempt: 7
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.211938 3781 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/wal
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/logs
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data/info.pb
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.65:0
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.177.65
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.65
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:12:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.259052 3936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb: Bootstrap starting.
03:12:25.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.274847 3936 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb: Neither blocks nor log segments found. Creating new log.
03:12:25.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.281774 3936 log.cc:824] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb: Log is configured to *not* fsync() on all Append() calls
03:12:25.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.298797 3936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb: No bootstrap required, opened a new log
03:12:25.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.354907 3936 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.356137 3936 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:12:25.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.356647 3936 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3143ff6434334d6cab70d9b84c7e07eb, State: Initialized, Role: FOLLOWER
03:12:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.357976 3936 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } }
03:12:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.370313 4063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:25.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.371191 4063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [sys.catalog]: This master's current role is: FOLLOWER
03:12:25.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.373674 3936 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [sys.catalog]: configured and running, proceeding with master startup.
03:12:25.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.374368 3909 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [term 0 FOLLOWER]: Advancing to term 1
03:12:25.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.378366 3911 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "15177b238f2e4b528936b63a613d85e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3143ff6434334d6cab70d9b84c7e07eb" is_pre_election: true
03:12:25.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.377430 3910 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "15177b238f2e4b528936b63a613d85e1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3143ff6434334d6cab70d9b84c7e07eb"
03:12:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.388842 3909 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [term 1 FOLLOWER]: Refusing update from remote peer 15177b238f2e4b528936b63a613d85e1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:12:25.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.390658 4045 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:12:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.471678 4074 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "15177b238f2e4b528936b63a613d85e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.471798 4063 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 15177b238f2e4b528936b63a613d85e1. Latest consensus state: current_term: 1 leader_uuid: "15177b238f2e4b528936b63a613d85e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.472581 4074 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [sys.catalog]: This master's current role is: FOLLOWER
03:12:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.472621 4063 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb [sys.catalog]: This master's current role is: FOLLOWER
03:12:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.489584 4079 catalog_manager.cc:1259] Loaded cluster ID: e60dc3f85f76445d9292ae46a6f24a2a
03:12:25.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.493836 4079 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb: loading cluster ID for follower catalog manager: success
03:12:25.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.511329 4079 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb: acquiring CA information for follower catalog manager: success
03:12:25.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.518553 4079 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3143ff6434334d6cab70d9b84c7e07eb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:12:25.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.808555 4045 sys_catalog.cc:455] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "15177b238f2e4b528936b63a613d85e1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "15177b238f2e4b528936b63a613d85e1" member_type: VOTER last_known_addr { host: "127.3.177.126" port: 40713 } } peers { permanent_uuid: "3143ff6434334d6cab70d9b84c7e07eb" member_type: VOTER last_known_addr { host: "127.3.177.125" port: 39077 } } peers { permanent_uuid: "aadaa136f8e64c899b8b06f4293db1f0" member_type: VOTER last_known_addr { host: "127.3.177.124" port: 45585 } } }
03:12:25.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:25.809751 4045 sys_catalog.cc:458] T 00000000000000000000000000000000 P 15177b238f2e4b528936b63a613d85e1 [sys.catalog]: This master's current role is: LEADER
03:12:25.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:25.872529 4061 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:25.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:25.873569 4061 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:25.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:25.874651 4061 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:25.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:25.933905 4061 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:25.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:25.935600 4061 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.177.65
03:12:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.000241 4061 tablet_server_runner.cc:78] Tablet server non-default flags:
03:12:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data
03:12:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/wal
03:12:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.65:0
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data/info.pb
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.65
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.177.65
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/logs
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.004406 4061 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.011777 4053 catalog_manager.cc:1259] Loaded cluster ID: e60dc3f85f76445d9292ae46a6f24a2a
03:12:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.012585 4053 catalog_manager.cc:1552] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: loading cluster ID for follower catalog manager: success
03:12:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.019660 4061 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:26.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.029974 4053 catalog_manager.cc:1574] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: acquiring CA information for follower catalog manager: success
03:12:26.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.036175 4053 catalog_manager.cc:1602] T 00000000000000000000000000000000 P aadaa136f8e64c899b8b06f4293db1f0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:12:26.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:26.093924 4089 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:26.106226 4061 server_base.cc:1034] running on GCE node
03:12:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:26.107277 4088 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:26.112843 4091 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.656476 4061 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:27.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.665339 4061 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:27.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.666829 4061 hybrid_clock.cc:648] HybridClock initialized: now 1790219547666793 us; error 51 us; skew 500 ppm
03:12:27.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.667933 4061 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:27.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.704308 4061 webserver.cc:460] Webserver started at http://127.3.177.65:35549/ using document root <none> and password file <none>
03:12:27.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.705662 4061 fs_manager.cc:359] Metadata directory not provided
03:12:27.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.705943 4061 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:27.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.706501 4061 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:27.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.713675 4061 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data/instance:
03:12:27.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6615076671e84089bace32b1bd955ad4"
03:12:27.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:27 on dist-test-slave-lltt"
03:12:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.715154 4061 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/wal/instance:
03:12:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6615076671e84089bace32b1bd955ad4"
03:12:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:27 on dist-test-slave-lltt"
03:12:27.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.749352 4061 fs_manager.cc:691] Time spent creating directory manager: real 0.033s user 0.018s sys 0.016s
03:12:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.764173 4098 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:27.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.765632 4061 fs_manager.cc:725] Time spent opening block manager: real 0.012s user 0.009s sys 0.000s
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.766052 4061 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/wal
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6615076671e84089bace32b1bd955ad4"
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:27 on dist-test-slave-lltt"
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.766480 4061 fs_report.cc:383] FS layout report
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/wal
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/wal
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data/data
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:27.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:27.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.888398 4061 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.890379 4061 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.890966 4061 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:27.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.905861 4061 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:12:27.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.922415 4061 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:12:27.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.922717 4061 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:12:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.923022 4061 ts_tablet_manager.cc:610] Registered 0 tablets
03:12:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:27.923262 4061 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:12:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.304924 4061 rpc_server.cc:307] RPC server started. Bound to: 127.3.177.65:42543
03:12:28.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.305073 4210 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.177.65:42543 every 8 connection(s)
03:12:28.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.308094 4061 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/data/info.pb
03:12:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.309182 3781 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4061
03:12:28.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.309700 3781 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-0/wal/instance
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.329608 3781 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/wal
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/logs
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data/info.pb
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:28.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.66:0
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.177.66
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.66
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:12:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.410709 4212 heartbeater.cc:347] Connected to a master server at 127.3.177.125:39077
03:12:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.411178 4212 heartbeater.cc:464] Registering TS with master...
03:12:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.411347 4216 heartbeater.cc:347] Connected to a master server at 127.3.177.126:40713
03:12:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.411708 4216 heartbeater.cc:464] Registering TS with master...
03:12:28.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.412364 4212 heartbeater.cc:511] Master 127.3.177.125:39077 requested a full tablet report, sending...
03:12:28.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.412705 4216 heartbeater.cc:511] Master 127.3.177.126:40713 requested a full tablet report, sending...
03:12:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.415890 3901 ts_manager.cc:194] Registered new tserver with Master: 6615076671e84089bace32b1bd955ad4 (127.3.177.65:42543)
03:12:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.416390 4211 heartbeater.cc:347] Connected to a master server at 127.3.177.124:45585
03:12:28.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.416702 4211 heartbeater.cc:464] Registering TS with master...
03:12:28.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.416975 3826 ts_manager.cc:194] Registered new tserver with Master: 6615076671e84089bace32b1bd955ad4 (127.3.177.65:42543)
03:12:28.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.417518 4211 heartbeater.cc:511] Master 127.3.177.124:45585 requested a full tablet report, sending...
03:12:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.420187 3826 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.177.65:56687
03:12:28.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.420321 3979 ts_manager.cc:194] Registered new tserver with Master: 6615076671e84089bace32b1bd955ad4 (127.3.177.65:42543)
03:12:28.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.717734 4215 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:28.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.718189 4215 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:28.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.718600 4215 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:28.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.749501 4215 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:28.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.750418 4215 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.177.66
03:12:28.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.786399 4215 tablet_server_runner.cc:78] Tablet server non-default flags:
03:12:28.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/wal
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.66:0
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data/info.pb
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.66
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.177.66
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/logs
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:28.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.787839 4215 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:28.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.789500 4215 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.809434 4227 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:29.425016 4216 heartbeater.cc:503] Master 127.3.177.126:40713 was elected leader, sending a full tablet report...
03:12:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.809460 4230 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:28.809434 4228 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:30.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:28.810546 4215 server_base.cc:1034] running on GCE node
03:12:30.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.026752 4215 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:30.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.028898 4215 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:30.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.030256 4215 hybrid_clock.cc:648] HybridClock initialized: now 1790219550030233 us; error 61 us; skew 500 ppm
03:12:30.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.031002 4215 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:30.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.038223 4215 webserver.cc:460] Webserver started at http://127.3.177.66:43313/ using document root <none> and password file <none>
03:12:30.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.039464 4215 fs_manager.cc:359] Metadata directory not provided
03:12:30.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.039693 4215 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:30.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.040251 4215 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.045723 4215 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data/instance:
03:12:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35f6e8ee79a04e2abeb0760932eabf12"
03:12:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:30 on dist-test-slave-lltt"
03:12:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.046844 4215 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/wal/instance:
03:12:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35f6e8ee79a04e2abeb0760932eabf12"
03:12:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:30 on dist-test-slave-lltt"
03:12:30.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.054319 4215 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:12:30.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.060669 4240 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.061970 4215 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:12:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.062341 4215 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/wal
03:12:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "35f6e8ee79a04e2abeb0760932eabf12"
03:12:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:30 on dist-test-slave-lltt"
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.062718 4215 fs_report.cc:383] FS layout report
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/wal
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/wal
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data/data
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:30.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.101250 4215 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:30.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.102674 4215 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:30.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.103099 4215 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:30.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.105693 4215 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:12:30.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.110491 4215 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:12:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.110692 4215 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:12:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.110888 4215 ts_tablet_manager.cc:610] Registered 0 tablets
03:12:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.111007 4215 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:12:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.267827 4215 rpc_server.cc:307] RPC server started. Bound to: 127.3.177.66:40909
03:12:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.267930 4352 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.177.66:40909 every 8 connection(s)
03:12:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.270361 4215 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/data/info.pb
03:12:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.280305 3781 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4215
03:12:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.281003 3781 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-1/wal/instance
03:12:30.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.308455 3781 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/wal
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/logs
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data/info.pb
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.67:0
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.177.67
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.67
03:12:30.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:30.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:12:30.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.314409 4354 heartbeater.cc:347] Connected to a master server at 127.3.177.125:39077
03:12:30.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.314851 4354 heartbeater.cc:464] Registering TS with master...
03:12:30.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.316054 4354 heartbeater.cc:511] Master 127.3.177.125:39077 requested a full tablet report, sending...
03:12:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.319690 3901 ts_manager.cc:194] Registered new tserver with Master: 35f6e8ee79a04e2abeb0760932eabf12 (127.3.177.66:40909)
03:12:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.324860 4353 heartbeater.cc:347] Connected to a master server at 127.3.177.124:45585
03:12:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.325178 4353 heartbeater.cc:464] Registering TS with master...
03:12:30.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.325935 4353 heartbeater.cc:511] Master 127.3.177.124:45585 requested a full tablet report, sending...
03:12:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.328611 4355 heartbeater.cc:347] Connected to a master server at 127.3.177.126:40713
03:12:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.328971 4355 heartbeater.cc:464] Registering TS with master...
03:12:30.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.328928 3979 ts_manager.cc:194] Registered new tserver with Master: 35f6e8ee79a04e2abeb0760932eabf12 (127.3.177.66:40909)
03:12:30.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.329829 4355 heartbeater.cc:511] Master 127.3.177.126:40713 requested a full tablet report, sending...
03:12:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.331916 3826 ts_manager.cc:194] Registered new tserver with Master: 35f6e8ee79a04e2abeb0760932eabf12 (127.3.177.66:40909)
03:12:30.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.334389 3826 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.177.66:46913
03:12:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.644142 4363 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.644614 4363 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.645133 4363 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.678467 4363 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:30.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.679315 4363 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.177.67
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.714839 4363 tablet_server_runner.cc:78] Tablet server non-default flags:
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.177.84:45521
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/wal
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.177.67:0
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data/info.pb
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.177.67
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.177.126:40713,127.3.177.125:39077,127.3.177.124:45585
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.177.67
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/logs
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:30.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.716077 4363 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:30.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.717901 4363 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:30.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.738615 4373 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:31.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.338106 4355 heartbeater.cc:503] Master 127.3.177.126:40713 was elected leader, sending a full tablet report...
03:12:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.738601 4370 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:30.738947 4371 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:31.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:30.742048 4363 server_base.cc:1034] running on GCE node
03:12:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.974448 4363 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:31.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.976790 4363 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.978205 4363 hybrid_clock.cc:648] HybridClock initialized: now 1790219551978173 us; error 44 us; skew 500 ppm
03:12:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.979065 4363 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:31.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.985462 4363 webserver.cc:460] Webserver started at http://127.3.177.67:44145/ using document root <none> and password file <none>
03:12:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.986333 4363 fs_manager.cc:359] Metadata directory not provided
03:12:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.986529 4363 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:31.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.986910 4363 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.991779 4363 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data/instance:
03:12:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f8707cc6a34488489c4cc5700123ad2"
03:12:31.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:31 on dist-test-slave-lltt"
03:12:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:31.992846 4363 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/wal/instance:
03:12:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f8707cc6a34488489c4cc5700123ad2"
03:12:31.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:31 on dist-test-slave-lltt"
03:12:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.000232 4363 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
03:12:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.005764 4381 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:32.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.006850 4363 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:12:32.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.007228 4363 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/wal
03:12:32.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f8707cc6a34488489c4cc5700123ad2"
03:12:32.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:31 on dist-test-slave-lltt"
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.007606 4363 fs_report.cc:383] FS layout report
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/wal
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/wal
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data/data
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:32.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:32.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.042165 4363 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:32.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.043716 4363 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:32.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.044268 4363 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:32.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.047508 4363 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:12:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.051864 4363 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:12:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.052125 4363 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:12:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.052472 4363 ts_tablet_manager.cc:610] Registered 0 tablets
03:12:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.052670 4363 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:12:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.219868 4363 rpc_server.cc:307] RPC server started. Bound to: 127.3.177.67:45989
03:12:32.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.219978 4493 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.177.67:45989 every 8 connection(s)
03:12:32.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.222563 4363 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/data/info.pb
03:12:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.232609 3781 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4363
03:12:32.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.233208 3781 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster5182838400673840902/ts-2/wal/instance
03:12:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.285470 4496 heartbeater.cc:347] Connected to a master server at 127.3.177.126:40713
03:12:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.286131 4496 heartbeater.cc:464] Registering TS with master...
03:12:32.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.287474 4496 heartbeater.cc:511] Master 127.3.177.126:40713 requested a full tablet report, sending...
03:12:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.290377 3826 ts_manager.cc:194] Registered new tserver with Master: 3f8707cc6a34488489c4cc5700123ad2 (127.3.177.67:45989)
03:12:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.290767 4495 heartbeater.cc:347] Connected to a master server at 127.3.177.125:39077
03:12:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.291141 4495 heartbeater.cc:464] Registering TS with master...
03:12:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.291844 4495 heartbeater.cc:511] Master 127.3.177.125:39077 requested a full tablet report, sending...
03:12:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.292026 3826 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.177.67:43175
03:12:32.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.294272 3901 ts_manager.cc:194] Registered new tserver with Master: 3f8707cc6a34488489c4cc5700123ad2 (127.3.177.67:45989)
03:12:32.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.298020 4494 heartbeater.cc:347] Connected to a master server at 127.3.177.124:45585
03:12:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.298341 4494 heartbeater.cc:464] Registering TS with master...
03:12:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.299101 4494 heartbeater.cc:511] Master 127.3.177.124:45585 requested a full tablet report, sending...
03:12:32.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.302248 3979 ts_manager.cc:194] Registered new tserver with Master: 3f8707cc6a34488489c4cc5700123ad2 (127.3.177.67:45989)
03:12:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.319584 3781 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:12:32.332 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:12:32.333 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:12:32.345 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.177.126"
port: 40713
}
bound_http_address {
host: "127.3.177.126"
port: 32963
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.177.125"
port: 39077
}
bound_http_address {
host: "127.3.177.125"
port: 34321
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.177.124"
port: 45585
}
bound_http_address {
host: "127.3.177.124"
port: 45497
}
}
}
03:12:32.351 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:12:32.353 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.177.65"
port: 42543
}
bound_http_address {
host: "127.3.177.65"
port: 35549
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.177.66"
port: 40909
}
bound_http_address {
host: "127.3.177.66"
port: 43313
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.177.67"
port: 45989
}
bound_http_address {
host: "127.3.177.67"
port: 44145
}
}
}
03:12:32.353 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:12:32.532 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:12:32.643 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskXqVMDO/test-tmp
03:12:32.644 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:12:32.644 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskXqVMDO/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:12:32.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:12:32.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:32.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:12:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.994637 4505 mini_oidc.cc:152] Starting JWKS server
03:12:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.998759 4505 webserver.cc:460] Webserver started at http://127.0.0.1:46551/ using document root <none> and password file <none>
03:12:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:32.999277 4505 mini_oidc.cc:189] Starting OIDC Discovery server
03:12:33.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:33.000985 4505 webserver.cc:460] Webserver started at http://127.0.0.1:41045/ using document root <none> and password file <none>
03:12:33.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:12:33.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/principal' for realm 'KRBTEST.COM',
03:12:33.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:12:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:33 dist-test-slave-lltt krb5kdc[4517](info): setting up network...
03:12:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:12:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:33 dist-test-slave-lltt krb5kdc[4517](info): set up 2 sockets
03:12:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:33 dist-test-slave-lltt krb5kdc[4517](info): commencing operation
03:12:33.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:12:33.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:33.298053 4496 heartbeater.cc:503] Master 127.3.177.126:40713 was elected leader, sending a full tablet report...
03:12:34.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.135618 4505 mini_kdc.cc:121] Time spent starting KDC: real 1.109s user 0.001s sys 0.019s
03:12:34.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:12:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:12:34.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:12:34.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:12:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:12:34.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:12:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:12:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:12:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/test-user.keytab.
03:12:34.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/test-user.keytab.
03:12:34.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:34 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219554, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:34.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:12:34.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:12:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:12:34.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:12:34Z Disabled control of system clock
03:12:34.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.102.126@KRBTEST.COM; defaulting to no policy
03:12:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:12:34.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.102.126@KRBTEST.COM" created.
03:12:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:12:34.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.102.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.126.keytab.
03:12:34.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.102.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.126.keytab.
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:34.342222 4505 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:34.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/logs
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/info.pb
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.126:36543
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.126
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.126.keytab
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.126
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:12:34.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:34.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:12:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.653949 4532 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.654702 4532 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:34.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.655282 4532 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:34.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.686862 4532 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:12:34.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.687173 4532 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:34.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.687590 4532 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.687927 4532 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.688247 4532 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.688556 4532 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.688830 4532 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46551/jwks.json
03:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:34.725049 4532 master_runner.cc:386] Master server non-default flags:
03:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.126.keytab
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.126
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.126:36543
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/info.pb
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.126
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/logs
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:34.727154 4532 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:34.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:34.728273 4532 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.748246 4541 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.748423 4538 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:36.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:34.748296 4539 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.079540 4540 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1328 milliseconds
03:12:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.079654 4532 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:12:36.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.081066 4532 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.084080 4532 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.085480 4532 hybrid_clock.cc:648] HybridClock initialized: now 1790219556085452 us; error 36 us; skew 500 ppm
03:12:36.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.086395 4532 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:36.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:36 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219556, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.093346 4532 init.cc:377] Logged in from keytab as kudu/127.4.102.126@KRBTEST.COM (short username kudu)
03:12:36.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.098526 4532 webserver.cc:460] Webserver started at http://127.4.102.126:37625/ using document root <none> and password file <none>
03:12:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.099401 4532 fs_manager.cc:359] Metadata directory not provided
03:12:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.099589 4532 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.099988 4532 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.104717 4532 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/instance:
03:12:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88f51cef020948eb86492c458ab9d4e6"
03:12:36.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:36 on dist-test-slave-lltt"
03:12:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.106050 4532 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal/instance:
03:12:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88f51cef020948eb86492c458ab9d4e6"
03:12:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:36 on dist-test-slave-lltt"
03:12:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.113718 4532 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:12:36.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.119467 4550 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:36.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.120594 4532 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:12:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.120980 4532 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88f51cef020948eb86492c458ab9d4e6"
03:12:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:36 on dist-test-slave-lltt"
03:12:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.121413 4532 fs_report.cc:383] FS layout report
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/data
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.186450 4532 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:36.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.192538 4532 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.193130 4532 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:36.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.281502 4532 rpc_server.cc:307] RPC server started. Bound to: 127.4.102.126:36543
03:12:36.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.281556 4601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.102.126:36543 every 8 connection(s)
03:12:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.285033 4532 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/info.pb
03:12:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.291478 4602 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:12:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.292781 4505 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4532
03:12:36.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.293275 4505 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal/instance
03:12:36.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.304191 4602 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } has no permanent_uuid. Determining permanent_uuid...
03:12:36.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.102.125@KRBTEST.COM; defaulting to no policy
03:12:36.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:12:36.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.102.125@KRBTEST.COM" created.
03:12:36.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:36 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219556, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.126@KRBTEST.COM for kudu/127.4.102.126@KRBTEST.COM
03:12:36.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:12:36.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.102.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.125.keytab.
03:12:36.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.102.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.125.keytab.
03:12:36.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.346539 4505 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/logs
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/info.pb
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.125:46499
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.125
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.125.keytab
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.125
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:36.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:12:36.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.354410 4602 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } has no permanent_uuid. Determining permanent_uuid...
03:12:36.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.356607 4554 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:36.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.361106 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.125:46499: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:36.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.410768 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } attempt: 1
03:12:36.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.414671 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.125:46499: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:36.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.483072 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } attempt: 2
03:12:36.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.486765 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.125:46499: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:36.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.578204 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } attempt: 3
03:12:36.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.582381 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.125:46499: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:36.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.672379 4608 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.673128 4608 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:36.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.674149 4608 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.710636 4608 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:12:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.710857 4608 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.711120 4608 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.711300 4608 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.711472 4608 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:36.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.711647 4608 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:36.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.711789 4608 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46551/jwks.json
03:12:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.725843 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } attempt: 4
03:12:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.729604 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.125:46499: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.748561 4608 master_runner.cc:386] Master server non-default flags:
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.125.keytab
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.125
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.125:46499
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/info.pb
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.125
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:36.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/logs
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.749818 4608 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:36.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.751051 4608 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:36.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.771272 4615 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:37.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:37.029037 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } attempt: 5
03:12:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:37.033401 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.125:46499: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:37.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:37.581022 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } attempt: 6
03:12:37.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:37.588505 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.125:46499: Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.771301 4616 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:36.771461 4618 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:36.774289 4608 server_base.cc:1034] running on GCE node
03:12:38.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.042613 4608 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:38.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.045738 4608 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:38.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.047235 4608 hybrid_clock.cc:648] HybridClock initialized: now 1790219558047178 us; error 65 us; skew 500 ppm
03:12:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.048139 4608 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:38 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219558, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:38.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.054814 4608 init.cc:377] Logged in from keytab as kudu/127.4.102.125@KRBTEST.COM (short username kudu)
03:12:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.060654 4608 webserver.cc:460] Webserver started at http://127.4.102.125:44335/ using document root <none> and password file <none>
03:12:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.061923 4608 fs_manager.cc:359] Metadata directory not provided
03:12:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.062178 4608 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.062695 4608 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.069696 4608 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/instance:
03:12:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "665b0adafbc24ad29a9a4d4bbbaf09db"
03:12:38.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:38 on dist-test-slave-lltt"
03:12:38.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.070744 4608 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal/instance:
03:12:38.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "665b0adafbc24ad29a9a4d4bbbaf09db"
03:12:38.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:38 on dist-test-slave-lltt"
03:12:38.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.078701 4608 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
03:12:38.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.084894 4628 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.086104 4608 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
03:12:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.086463 4608 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "665b0adafbc24ad29a9a4d4bbbaf09db"
03:12:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:38 on dist-test-slave-lltt"
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.086823 4608 fs_report.cc:383] FS layout report
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/data
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:38.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:38.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.152091 4608 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:38.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.158082 4608 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:38.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.158630 4608 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.239315 4608 rpc_server.cc:307] RPC server started. Bound to: 127.4.102.125:46499
03:12:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.239391 4679 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.102.125:46499 every 8 connection(s)
03:12:38.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.242254 4608 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/info.pb
03:12:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.247315 4505 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4608
03:12:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.247745 4505 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal/instance
03:12:38.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.248145 4680 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:12:38.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.102.124@KRBTEST.COM; defaulting to no policy
03:12:38.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.264693 4680 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } has no permanent_uuid. Determining permanent_uuid...
03:12:38.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:12:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.102.124@KRBTEST.COM" created.
03:12:38.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:12:38.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.102.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.124.keytab.
03:12:38.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.102.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.124.keytab.
03:12:38.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:38 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219558, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.125@KRBTEST.COM for kudu/127.4.102.126@KRBTEST.COM
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.311460 4505 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/logs
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/info.pb
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.124:38493
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.124
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:38.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.124.keytab
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.124
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:38.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:12:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.330202 4680 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } has no permanent_uuid. Determining permanent_uuid...
03:12:38.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:38 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219558, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.125@KRBTEST.COM for kudu/127.4.102.125@KRBTEST.COM
03:12:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.365684 4680 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } has no permanent_uuid. Determining permanent_uuid...
03:12:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.368287 4632 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.372623 4680 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.422166 4680 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 1
03:12:38.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.426436 4680 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.494931 4680 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 2
03:12:38.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.498922 4680 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.590327 4680 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 3
03:12:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.594535 4680 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.649106 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } attempt: 7
03:12:38.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.660187 4686 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:38.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.661185 4686 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:38.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.662654 4686 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:38.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:38 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219556, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.126@KRBTEST.COM for kudu/127.4.102.125@KRBTEST.COM
03:12:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.677588 4602 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } has no permanent_uuid. Determining permanent_uuid...
03:12:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.683370 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.701963 4686 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:12:38.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.702239 4686 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:38.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.702625 4686 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.702942 4686 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.703243 4686 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.703557 4686 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:38.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.703783 4686 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46551/jwks.json
03:12:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.737923 4680 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 4
03:12:38.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.742270 4680 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.742303 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 1
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.743855 4686 master_runner.cc:386] Master server non-default flags:
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.124.keytab
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.124
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.124:38493
03:12:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/info.pb
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.124
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:38.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/logs
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.745301 4686 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.746099 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.746483 4686 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:38.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.766752 4697 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:38.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.827472 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 2
03:12:38.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.832182 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.917620 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 3
03:12:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.922421 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:39.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:39.041699 4680 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 5
03:12:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:39.046062 4680 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:39.062835 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 4
03:12:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:39.067865 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:39.351279 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 5
03:12:39.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:39.355978 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:39.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:39.593474 4680 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 6
03:12:39.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:39.600965 4680 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:39.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:39.908579 4602 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } attempt: 6
03:12:39.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:39.915663 4602 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.102.124:38493: Network error: Client connection negotiation failed: client connection to 127.4.102.124:38493: connect: Connection refused (error 111)
03:12:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:38.767573 4686 server_base.cc:1034] running on GCE node
03:12:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.766773 4698 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:40.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:38.769575 4700 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:40.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.040012 4686 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:40.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.043675 4686 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:40.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.045248 4686 hybrid_clock.cc:648] HybridClock initialized: now 1790219560045144 us; error 76 us; skew 500 ppm
03:12:40.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.046155 4686 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:40.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219560, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:40.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.052738 4686 init.cc:377] Logged in from keytab as kudu/127.4.102.124@KRBTEST.COM (short username kudu)
03:12:40.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.059350 4686 webserver.cc:460] Webserver started at http://127.4.102.124:41605/ using document root <none> and password file <none>
03:12:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.060464 4686 fs_manager.cc:359] Metadata directory not provided
03:12:40.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.060748 4686 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:40.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.061404 4686 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:12:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.067179 4686 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/instance:
03:12:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e26fc1f7ef4d899a9173a045d5224f"
03:12:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:40 on dist-test-slave-lltt"
03:12:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.068434 4686 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal/instance:
03:12:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e26fc1f7ef4d899a9173a045d5224f"
03:12:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:40 on dist-test-slave-lltt"
03:12:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.077827 4686 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.011s sys 0.000s
03:12:40.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.085062 4714 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:40.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.086498 4686 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:12:40.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.086926 4686 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:40.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e26fc1f7ef4d899a9173a045d5224f"
03:12:40.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:40 on dist-test-slave-lltt"
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.087371 4686 fs_report.cc:383] FS layout report
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/data
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:40.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:40.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.132170 4686 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.138260 4686 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.138674 4686 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.225723 4686 rpc_server.cc:307] RPC server started. Bound to: 127.4.102.124:38493
03:12:40.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.225798 4765 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.102.124:38493 every 8 connection(s)
03:12:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.229437 4686 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/info.pb
03:12:40.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.236884 4766 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:12:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.240054 4505 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4686
03:12:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.240567 4505 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal/instance
03:12:40.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.269054 4766 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } has no permanent_uuid. Determining permanent_uuid...
03:12:40.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219554, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.102.124@KRBTEST.COM
03:12:40.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219560, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for kudu/127.4.102.126@KRBTEST.COM
03:12:40.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.319154 4766 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } has no permanent_uuid. Determining permanent_uuid...
03:12:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219554, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.102.125@KRBTEST.COM
03:12:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219560, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for kudu/127.4.102.125@KRBTEST.COM
03:12:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.345876 4766 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } has no permanent_uuid. Determining permanent_uuid...
03:12:40.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219560, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for kudu/127.4.102.124@KRBTEST.COM
03:12:40.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219554, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.102.126@KRBTEST.COM
03:12:40.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.380885 4505 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
03:12:40.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.381433 4766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Bootstrap starting.
03:12:40.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.388913 4766 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Neither blocks nor log segments found. Creating new log.
03:12:40.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.391086 4766 log.cc:824] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Log is configured to *not* fsync() on all Append() calls
03:12:40.391 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:40.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:12:40.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.396461 4766 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: No bootstrap required, opened a new log
03:12:40.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:12:40.403 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:12:40.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.102.126"
port: 36543
}
bound_http_address {
host: "127.4.102.126"
port: 37625
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.102.125"
port: 46499
}
bound_http_address {
host: "127.4.102.125"
port: 44335
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.102.124"
port: 38493
}
bound_http_address {
host: "127.4.102.124"
port: 41605
}
}
}
03:12:40.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:12:40.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:40.408 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:12:40.410 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:12:40.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.421949 4766 raft_consensus.cc:348] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:40.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.422811 4766 raft_consensus.cc:374] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:12:40.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.423118 4766 raft_consensus.cc:729] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33e26fc1f7ef4d899a9173a045d5224f, State: Initialized, Role: FOLLOWER
03:12:40.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.423938 4766 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.426783 4776 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } } }
03:12:40.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.428172 4776 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [sys.catalog]: This master's current role is: FOLLOWER
03:12:40.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.428678 4766 sys_catalog.cc:564] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [sys.catalog]: configured and running, proceeding with master startup.
03:12:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219560, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:12:40.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:40.453 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc
03:12:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.458539 4788 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:40.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.459105 4788 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:40.460 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:12:40.461 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc
03:12:40.463 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:12:40.467 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:12:40.470 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:12:40.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:12:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.453980 4686 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: OK, instance_id { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" instance_seqno: 1790219556257688 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:12:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.492185 4686 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:12:40.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:40 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219560, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:40.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:12:40.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:40.505 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc
03:12:40.507 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:12:40.507 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.102.125:46499
03:12:40.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:12:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.509984 4505 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 4608
03:12:40.533 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:40.533 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.102.124:38493
03:12:40.533 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:12:40.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.534075 4505 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 4686
03:12:40.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:40.560 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.102.126:36543
03:12:40.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:12:40.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.560721 4505 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 4532
03:12:40.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:40.583 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.102.125:46499
03:12:40.586 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.587673 4505 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/logs
03:12:40.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/info.pb
03:12:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.125:46499
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.125
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44335
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.125.keytab
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.125
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:12:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.917659 4792 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:40.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.918392 4792 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.919135 4792 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.952229 4792 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:12:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.952525 4792 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.952913 4792 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.953285 4792 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.953596 4792 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.953912 4792 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:40.954162 4792 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46551/jwks.json
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.991833 4792 master_runner.cc:386] Master server non-default flags:
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.125.keytab
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.125
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.125:46499
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:40.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:12:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:12:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:40.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/info.pb
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.125
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44335
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/logs
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:40.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.993363 4792 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:40.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:40.994753 4792 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:41.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:41.014361 4800 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:41.014415 4802 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:41.016330 4792 server_base.cc:1034] running on GCE node
03:12:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:41.014802 4799 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:42.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.278012 4792 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:42.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.281291 4792 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.282776 4792 hybrid_clock.cc:648] HybridClock initialized: now 1790219562282725 us; error 67 us; skew 500 ppm
03:12:42.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.283674 4792 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:42.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:42 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219562, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:42.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.291760 4792 init.cc:377] Logged in from keytab as kudu/127.4.102.125@KRBTEST.COM (short username kudu)
03:12:42.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.297282 4792 webserver.cc:460] Webserver started at http://127.4.102.125:44335/ using document root <none> and password file <none>
03:12:42.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.298223 4792 fs_manager.cc:359] Metadata directory not provided
03:12:42.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.298432 4792 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:42.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.308184 4792 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.003s sys 0.003s
03:12:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.314167 4811 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:42.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.315572 4792 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:12:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.315888 4792 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "665b0adafbc24ad29a9a4d4bbbaf09db"
03:12:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:38 on dist-test-slave-lltt"
03:12:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.318419 4792 fs_report.cc:383] FS layout report
03:12:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:42.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/data
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:42.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.360684 4792 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.367187 4792 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:42.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.367636 4792 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.443354 4792 rpc_server.cc:307] RPC server started. Bound to: 127.4.102.125:46499
03:12:42.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.443468 4862 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.102.125:46499 every 8 connection(s)
03:12:42.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.446095 4792 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/data/info.pb
03:12:42.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.453948 4505 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4792
03:12:42.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:42.454 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.102.124:38493
03:12:42.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.456156 4505 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/logs
03:12:42.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/info.pb
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.124:38493
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.124
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41605
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.124.keytab
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.124
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.457309 4863 sys_catalog.cc:263] Verifying existing consensus state
03:12:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 03:12:42.458340 4863 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:12:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.458818 4792 master.cc:537] Master@127.4.102.125:46499 shutting down...
03:12:42.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.530385 4792 master.cc:552] Master@127.4.102.125:46499 shutdown complete.
03:12:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:12:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.799701 4865 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.800231 4865 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.800812 4865 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:42.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.837512 4865 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:12:42.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.837742 4865 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:42.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.838023 4865 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:42.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.838220 4865 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:42.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.838411 4865 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:42.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.838605 4865 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:42.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.838744 4865 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46551/jwks.json
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.892867 4865 master_runner.cc:386] Master server non-default flags:
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:42.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.124.keytab
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.124
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.124:38493
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/info.pb
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.124
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41605
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/logs
03:12:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:42.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.894317 4865 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:42.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:42.895709 4865 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:42.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.914206 4872 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:44.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:42.914314 4871 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:44.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.152813 4873 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:12:44.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.154407 4874 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.154512 4865 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:12:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.157608 4865 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:44.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.160076 4865 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:44.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.161484 4865 hybrid_clock.cc:648] HybridClock initialized: now 1790219564161442 us; error 39 us; skew 500 ppm
03:12:44.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.162465 4865 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:44.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:44 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219564, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.174521 4865 init.cc:377] Logged in from keytab as kudu/127.4.102.124@KRBTEST.COM (short username kudu)
03:12:44.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.179855 4865 webserver.cc:460] Webserver started at http://127.4.102.124:41605/ using document root <none> and password file <none>
03:12:44.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.180878 4865 fs_manager.cc:359] Metadata directory not provided
03:12:44.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.181156 4865 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:44.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.189531 4865 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
03:12:44.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.194752 4884 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:44.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.196156 4865 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.196441 4865 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e26fc1f7ef4d899a9173a045d5224f"
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:40 on dist-test-slave-lltt"
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.199018 4865 fs_report.cc:383] FS layout report
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/wal
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/data
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:44.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:44.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.242516 4865 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.248242 4865 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.248723 4865 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:44.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.328629 4865 rpc_server.cc:307] RPC server started. Bound to: 127.4.102.124:38493
03:12:44.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.328714 4935 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.102.124:38493 every 8 connection(s)
03:12:44.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.331756 4865 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-2/data/info.pb
03:12:44.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.337069 4505 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4865
03:12:44.338 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:44.339 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.102.126:36543
03:12:44.339 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.341113 4505 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:12:44.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/logs
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/info.pb
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:12:44.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.126:36543
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.126
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.126.keytab
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.126
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:12:44.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.343724 4936 sys_catalog.cc:263] Verifying existing consensus state
03:12:44.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.350543 4936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Bootstrap starting.
03:12:44.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.391786 4936 log.cc:824] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Log is configured to *not* fsync() on all Append() calls
03:12:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.397708 4936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:12:44.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.398578 4936 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Bootstrap complete.
03:12:44.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.435087 4936 raft_consensus.cc:348] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:44.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.439333 4936 raft_consensus.cc:374] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:12:44.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.439642 4936 raft_consensus.cc:729] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33e26fc1f7ef4d899a9173a045d5224f, State: Initialized, Role: FOLLOWER
03:12:44.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.440596 4936 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:44.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.443334 4940 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } } }
03:12:44.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.444002 4940 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [sys.catalog]: This master's current role is: FOLLOWER
03:12:44.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.445174 4936 sys_catalog.cc:564] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [sys.catalog]: configured and running, proceeding with master startup.
03:12:44.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.476015 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.476732 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.479372 4865 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111),
03:12:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.504240 4890 tablet.cc:2369] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:12:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.504619 4865 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:12:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.722182 4938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:12:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.722723 4938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:12:44.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.723294 4938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:12:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.755894 4938 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:12:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.756117 4938 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:12:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.756376 4938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:12:44.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.756654 4938 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:12:44.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.756868 4938 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:12:44.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.757054 4938 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:12:44.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.757239 4938 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46551/jwks.json
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.795598 4938 master_runner.cc:386] Master server non-default flags:
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.102.84:43125
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.102.126:36543,127.4.102.125:46499,127.4.102.124:38493
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/krb5kdc/kudu_127.4.102.126.keytab
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.102.126
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.102.126:36543
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46551/jwks.json
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:12:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/info.pb
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.102.126
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37625
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/logs
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:12:44.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 02:28:12 UTC on ccycloud.quasar-rlwkeb.root.comops.site
03:12:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669249
03:12:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:12:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.797072 4938 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:12:44.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.798481 4938 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:12:44.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.817909 4960 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.869732 4940 raft_consensus.cc:482] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:12:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.870298 4940 raft_consensus.cc:504] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:44.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.874997 4888 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:12:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.877342 4940 leader_election.cc:288] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543), 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499)
03:12:44.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.890453 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543): Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:12:44.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.893106 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499): Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:44.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.893563 4888 leader_election.cc:302] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 33e26fc1f7ef4d899a9173a045d5224f; no voters: 665b0adafbc24ad29a9a4d4bbbaf09db, 88f51cef020948eb86492c458ab9d4e6
03:12:44.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.894443 4940 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:12:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:45.483991 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:45.484516 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:46.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:44.820070 4938 server_base.cc:1034] running on GCE node
03:12:46.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.818055 4962 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:46.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:44.817948 4959 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:12:46.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.151273 4938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:12:46.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.156172 4938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:12:46.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.157702 4938 hybrid_clock.cc:648] HybridClock initialized: now 1790219566157628 us; error 89 us; skew 500 ppm
03:12:46.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.158540 4938 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:12:46.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:46 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219566, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:46.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.165151 4938 init.cc:377] Logged in from keytab as kudu/127.4.102.126@KRBTEST.COM (short username kudu)
03:12:46.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.171293 4938 webserver.cc:460] Webserver started at http://127.4.102.126:37625/ using document root <none> and password file <none>
03:12:46.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.172199 4938 fs_manager.cc:359] Metadata directory not provided
03:12:46.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.172405 4938 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:12:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.181449 4938 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
03:12:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.186537 4972 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:12:46.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.187791 4938 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:12:46.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.188165 4938 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data,/tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88f51cef020948eb86492c458ab9d4e6"
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 03:12:36 on dist-test-slave-lltt"
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.190950 4938 fs_report.cc:383] FS layout report
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/data
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:12:46.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:12:46.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:12:46.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.230172 4938 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:12:46.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.236411 4938 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:12:46.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.236953 4938 kserver.cc:163] Server-wide thread pool size limit: 3276
03:12:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.329512 4938 rpc_server.cc:307] RPC server started. Bound to: 127.4.102.126:36543
03:12:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.329663 5023 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.102.126:36543 every 8 connection(s)
03:12:46.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.333598 4938 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/data/info.pb
03:12:46.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.336352 4505 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu as pid 4938
03:12:46.337 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:12:46.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.359549 5024 sys_catalog.cc:263] Verifying existing consensus state
03:12:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 03:12:46.361233 5024 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:12:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.363515 4938 master.cc:537] Master@127.4.102.126:36543 shutting down...
03:12:46.412 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.449718 4938 master.cc:552] Master@127.4.102.126:36543 shutdown complete.
03:12:46.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskXqVMDO/test-tmp/mini-kudu-cluster6094526356818529920/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:12:46.482 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.102.124:38493(127.4.102.124:38493)] connecting to peer
03:12:46.484 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:46.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:46.488126 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:46.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:46.488477 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:46.542 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.4.102.124:38493(127.4.102.124:38493)] Successfully connected to peer
03:12:46.545 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.548 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.551 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.617 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:12:46.625 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.102.124
03:12:46.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.783324 5030 raft_consensus.cc:482] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:12:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.783723 5030 raft_consensus.cc:504] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:46.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.785560 5030 leader_election.cc:288] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543), 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499)
03:12:46.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:46.795287 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543): Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:12:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:46.795964 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499): Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:46.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.796357 4888 leader_election.cc:302] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 33e26fc1f7ef4d899a9173a045d5224f; no voters: 665b0adafbc24ad29a9a4d4bbbaf09db, 88f51cef020948eb86492c458ab9d4e6
03:12:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:46.797092 5030 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:12:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:46 dist-test-slave-lltt krb5kdc[4517](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219560, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.102.124@KRBTEST.COM
03:12:46.841 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa16a0137, L:/127.0.0.1:37122 - R:/127.4.102.124:38493] using SASL/GSSAPI
03:12:46.858 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:46.863 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:46.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:46.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.872 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.872 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.873 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.875 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:46.883 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:46.887 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:46.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.890 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.890 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.891 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.893 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:46.902 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:46.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:46.906 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.907 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.907 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.910 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.912 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:46.922 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:46.924 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:46.926 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.926 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.927 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.928 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.928 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.930 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:46.962 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:46.964 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:46.967 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.967 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.967 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.967 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.968 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:46.968 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:46.970 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:47.043 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:47.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:47.047 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.047 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.048 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.050 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:47.123 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:47.125 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:47.126 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.127 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.127 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.128 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.128 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.130 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:47.243 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:47.244 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:47.246 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.247 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.248 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.248 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.248 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.250 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:47.491341 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:47.491587 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:47.622 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:47.624 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:47.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.627 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:47.627 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.627 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:47.629 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:48.143 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:48.144 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:48.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:48.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:48.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:48.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:48.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:48.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:48.149 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:48.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:48.494455 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:48.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:48.494832 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:49.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:49.279186 5036 raft_consensus.cc:482] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:12:49.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:49.279645 5036 raft_consensus.cc:504] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:49.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:49.281415 5036 leader_election.cc:288] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543), 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499)
03:12:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:49.287737 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543): Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:12:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:49.288326 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499): Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:49.288581 4888 leader_election.cc:302] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 33e26fc1f7ef4d899a9173a045d5224f; no voters: 665b0adafbc24ad29a9a4d4bbbaf09db, 88f51cef020948eb86492c458ab9d4e6
03:12:49.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:49.289088 5036 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:12:49.463 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:49.464 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:49.466 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:49.467 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:49.467 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:49.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:49.467 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:49.467 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:49.468 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:49.497920 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:49.498183 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:49.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:49.507367 4865 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111),
03:12:49.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:49.507791 4865 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:12:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:50.501017 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:50.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:50.501312 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:50.816799 4010 debug-util.cc:398] Leaking SignalData structure 0x7b08000826c0 after lost signal to thread 3943
03:12:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:50.817507 4010 debug-util.cc:398] Leaking SignalData structure 0x7b0800082d20 after lost signal to thread 4013
03:12:51.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:51.504109 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:51.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:51.504443 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:52.283 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:52.284 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:52.286 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.287 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.287 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.287 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.288 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.288 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:52.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:52.396569 5043 raft_consensus.cc:482] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:12:52.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:52.396976 5043 raft_consensus.cc:504] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:52.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:52.398627 5043 leader_election.cc:288] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543), 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499)
03:12:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:52.402463 4888 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111) [suppressed 5 similar messages]
03:12:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:52.405735 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543): Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:12:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:52.406342 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499): Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:52.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:52.406605 4888 leader_election.cc:302] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 33e26fc1f7ef4d899a9173a045d5224f; no voters: 665b0adafbc24ad29a9a4d4bbbaf09db, 88f51cef020948eb86492c458ab9d4e6
03:12:52.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:52.407181 5043 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:12:52.442 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:52.444 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:52.445 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.445 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.446 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.446 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.449 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:52.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:52.507292 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:52.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:52.507606 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:52.823 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:52.824 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:52.826 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.826 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.826 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.826 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.827 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:52.827 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:52.829 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:53.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:53.364 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:53.366 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:53.367 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:53.367 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:53.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:53.367 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:53.367 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:53.369 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:53.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:53.511042 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:53.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:53.511368 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:54.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:12:54 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219574, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:12:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:54.178401 4880 init.cc:295] Successfully reacquired a new kerberos TGT
03:12:54.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:54.510840 4865 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111),
03:12:54.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:54.511238 4865 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:12:54.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:54.514648 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:54.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:54.514874 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:55.142 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:55.144 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:55.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:55.146 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:55.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:55.146 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:55.146 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:55.146 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:55.148 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:55.517813 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:55.518054 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:56.003 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:56.004 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:56.006 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:56.006 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:56.006 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:56.006 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:56.006 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:56.006 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:56.008 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:56.521008 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:56.521281 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:57.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:57.524138 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:57.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:57.524402 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:57.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:57.589799 5052 raft_consensus.cc:482] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:12:57.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:57.590252 5052 raft_consensus.cc:504] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:12:57.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:57.591866 5052 leader_election.cc:288] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543), 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499)
03:12:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:57.595007 4888 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:12:57.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:57.598418 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543): Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:12:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:57.599069 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499): Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:12:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:57.599462 4888 leader_election.cc:302] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 33e26fc1f7ef4d899a9173a045d5224f; no voters: 665b0adafbc24ad29a9a4d4bbbaf09db, 88f51cef020948eb86492c458ab9d4e6
03:12:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:57.600188 5052 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:12:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:58.527236 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:58.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:58.527555 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:12:59.222 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:12:59.224 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:12:59.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:59.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:59.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:12:59.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:59.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:59.228 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:12:59.229 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:12:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:59.513880 4865 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111),
03:12:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:12:59.514248 4865 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:12:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:59.530295 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:12:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:12:59.530544 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:00.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:00.533298 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:00.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:00.533555 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:01.536409 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:01.536660 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:02.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:02.539417 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:02.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:02.539665 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:02.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:02.638180 5064 raft_consensus.cc:482] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:13:02.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:02.638612 5064 raft_consensus.cc:504] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:13:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:02.640455 5064 leader_election.cc:288] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543), 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499)
03:13:02.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:02.642939 4888 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:13:02.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:02.646154 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543): Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:13:02.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:02.646844 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499): Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:13:02.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:02.647111 4888 leader_election.cc:302] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 33e26fc1f7ef4d899a9173a045d5224f; no voters: 665b0adafbc24ad29a9a4d4bbbaf09db, 88f51cef020948eb86492c458ab9d4e6
03:13:02.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:02.647728 5064 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:13:03.263 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:13:03.264 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:13:03.266 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:03.266 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:03.266 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:03.266 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:03.266 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:03.266 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:03.269 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:13:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:03.542910 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:03.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:03.543272 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:13:04 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219584, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:13:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:04.180917 4880 init.cc:295] Successfully reacquired a new kerberos TGT
03:13:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:04.517410 4865 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111),
03:13:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:04.517882 4865 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:13:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:04.546154 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:04.546450 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:04.863 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:13:04.864 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:04.864 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:13:04.864 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:04.865 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:04.867 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:04.867 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:04.867 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:04.869 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:13:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:05.549495 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:05.549733 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:06.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:06.552482 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:06.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:06.552742 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:07.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:07.555590 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:07.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:07.555898 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:08.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:08.559024 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:08.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:08.559258 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:08.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:13:08.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:13:08.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:08.905 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:08.905 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:08.905 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:08.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:08.906 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:08.908 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:13:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:09.520582 4865 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111),
03:13:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:09.521018 4865 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:13:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:09.562158 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:09.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:09.562435 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:10.565335 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:10.565595 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:11.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:13:11.364 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:13:11.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:11.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:11.365 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:11.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:11.365 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:11.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:11.367 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:13:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:11.568307 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:11.568576 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:12.571321 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:12.571575 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:13.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:13.575188 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:13.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:13.575524 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:13.772949 4490 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6640 after lost signal to thread 4365
03:13:13.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:13.773797 4490 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6de0 after lost signal to thread 4493
03:13:14.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 03:13:14 dist-test-slave-lltt krb5kdc[4517](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790219594, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.102.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:13:14.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:14.183636 4880 init.cc:295] Successfully reacquired a new kerberos TGT
03:13:14.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:14.524369 4865 master_runner.cc:186] Error getting master registration for 127.4.102.126:36543: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111),
03:13:14.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:14.524739 4865 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:13:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:14.578481 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:14.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:14.578711 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:15.222 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:13:15.223 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:15.223 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:15.224 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:15.224 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:13:15.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:15.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:15.226 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:15.228 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:13:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:15.581281 4951 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:13:15.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:15.581605 4951 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:13:15.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:15.707929 5084 raft_consensus.cc:482] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:13:15.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:15.708387 5084 raft_consensus.cc:504] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88f51cef020948eb86492c458ab9d4e6" member_type: VOTER last_known_addr { host: "127.4.102.126" port: 36543 } } peers { permanent_uuid: "665b0adafbc24ad29a9a4d4bbbaf09db" member_type: VOTER last_known_addr { host: "127.4.102.125" port: 46499 } } peers { permanent_uuid: "33e26fc1f7ef4d899a9173a045d5224f" member_type: VOTER last_known_addr { host: "127.4.102.124" port: 38493 } }
03:13:15.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:15.710115 5084 leader_election.cc:288] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543), 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499)
03:13:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:15.713223 4888 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:13:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:15.716444 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88f51cef020948eb86492c458ab9d4e6 (127.4.102.126:36543): Network error: Client connection negotiation failed: client connection to 127.4.102.126:36543: connect: Connection refused (error 111)
03:13:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:15.717074 4888 leader_election.cc:334] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 665b0adafbc24ad29a9a4d4bbbaf09db (127.4.102.125:46499): Network error: Client connection negotiation failed: client connection to 127.4.102.125:46499: connect: Connection refused (error 111)
03:13:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:15.717507 4888 leader_election.cc:302] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 33e26fc1f7ef4d899a9173a045d5224f; no voters: 665b0adafbc24ad29a9a4d4bbbaf09db, 88f51cef020948eb86492c458ab9d4e6
03:13:15.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:15.718276 5084 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 33e26fc1f7ef4d899a9173a045d5224f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:13:16.103 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.125:46499(127.4.102.125:46499)] connecting to peer
03:13:16.104 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.102.126:36543(127.4.102.126:36543)] connecting to peer
03:13:16.105 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:16.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:16.105 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.102.126:36543(127.4.102.126:36543)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:16.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.125:46499(127.4.102.125:46499)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:16.105 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.126:36543: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543
03:13:16.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.102.125:46499: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
03:13:16.107 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543; will retry
03:13:16.111 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29766), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [195ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [198ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [502ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [510ms] refreshing cache from master, [511ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [520ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [522ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [523ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [530ms] refreshing cache from master, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [532ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [538ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [539ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [540ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [541ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [550ms] refreshing cache from master, [550ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [551ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [552ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [555ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [557ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [559ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [560ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [570ms] refreshing cache from master, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [571ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [575ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [576ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [578ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [610ms] refreshing cache from master, [610ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [611ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [617ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [618ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [690ms] refreshing cache from master, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [692ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [693ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [698ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [699ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [770ms] refreshing cache from master, [771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [774ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [775ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [777ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [778ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [890ms] refreshing cache from master, [891ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [895ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [896ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [897ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [898ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1270ms] refreshing cache from master, [1270ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1271ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1272ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1274ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1275ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1276ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1277ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1790ms] refreshing cache from master, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1794ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1796ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1797ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1797ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3110ms] refreshing cache from master, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [3112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3116ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [3117ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [5930ms] refreshing cache from master, [5930ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [5931ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [5932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [5935ms] too many traces: truncated at 100 traces, deferred=Deferred@334932688(state=PENDING, result=null, callback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156), errback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156)))
03:13:16.116 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.102.125:46499(127.4.102.125:46499)] disconnecting while in state TERMINATED
03:13:16.117 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.102.126:36543(127.4.102.126:36543)] disconnecting while in state TERMINATED
03:13:16.117 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.102.124:38493(127.4.102.124:38493)] disconnecting while in state READY
03:13:16.122 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:13:16.124 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:13:16.124 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.4.102.124:38493(127.4.102.124:38493)] handling channelInactive
03:13:16.124 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.102.124:38493(127.4.102.124:38493)] cleaning up while in state READY due to: connection closed
03:13:16.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:16.130463 4505 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 4865
03:13:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:13:17Z chronyd exiting
03:13:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:18.164530 4349 debug-util.cc:398] Leaking SignalData structure 0x7b08000bc260 after lost signal to thread 4222
03:13:18.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 03:13:18.165777 4349 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcfa0 after lost signal to thread 4352
03:13:18.365 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:13:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:18.366567 3781 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 4061
03:13:18.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:18.403255 3781 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 4215
03:13:18.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:18.433607 3781 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 4363
03:13:18.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:18.467681 3781 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 3793
03:13:18.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:18.501683 3781 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 3863
03:13:18.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 03:13:18.543005 3781 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskXqVMDO/build/tsan/bin/kudu with pid 3938
03:13:19.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T03:13:19Z chronyd exiting
03:13:20.738 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29766), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [195ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [198ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [502ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [510ms] refreshing cache from master, [511ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [520ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [522ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [523ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [530ms] refreshing cache from master, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [532ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [538ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [539ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [540ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [541ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [550ms] refreshing cache from master, [550ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [551ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [552ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [555ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [557ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [559ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [560ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [570ms] refreshing cache from master, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [571ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [575ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [576ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [578ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [610ms] refreshing cache from master, [610ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [611ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [617ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [618ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [690ms] refreshing cache from master, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [692ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [693ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [698ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [699ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [770ms] refreshing cache from master, [771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [774ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [775ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [777ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [778ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [890ms] refreshing cache from master, [891ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [895ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [896ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [897ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [898ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1270ms] refreshing cache from master, [1270ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1271ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1272ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1274ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1275ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1276ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1277ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1790ms] refreshing cache from master, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1794ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1796ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1797ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1797ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3110ms] refreshing cache from master, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [3112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3116ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [3117ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [5930ms] refreshing cache from master, [5930ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [5931ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [5932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [5935ms] too many traces: truncated at 100 traces, deferred=Deferred@334932688(state=PENDING, result=null, callback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156), errback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.102.125:46499
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29766), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [195ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [198ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [502ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [510ms] refreshing cache from master, [511ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [520ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [522ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [523ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [530ms] refreshing cache from master, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [532ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [538ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [539ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [540ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [541ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [550ms] refreshing cache from master, [550ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [551ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [552ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [555ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [557ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [559ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [560ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [570ms] refreshing cache from master, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [571ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [575ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [576ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [578ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [610ms] refreshing cache from master, [610ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [611ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [617ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [618ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [690ms] refreshing cache from master, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [692ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [693ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [698ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [699ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [770ms] refreshing cache from master, [771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [774ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [775ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [777ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [778ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [890ms] refreshing cache from master, [891ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [895ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [896ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [897ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [898ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1270ms] refreshing cache from master, [1270ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1271ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1272ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1274ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1275ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1276ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1277ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1790ms] refreshing cache from master, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1794ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1796ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1797ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1797ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3110ms] refreshing cache from master, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [3112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3116ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [3117ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [5930ms] refreshing cache from master, [5930ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [5931ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [5932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [5935ms] too many traces: truncated at 100 traces, deferred=Deferred@334932688(state=PENDING, result=null, callback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156), errback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.102.125:46499
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
03:13:22.022 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:13:22.022 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29766), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [195ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [198ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [502ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [510ms] refreshing cache from master, [511ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [520ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [522ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [523ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [530ms] refreshing cache from master, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [532ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [538ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [539ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [540ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [541ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [550ms] refreshing cache from master, [550ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [551ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [552ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [555ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [557ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [559ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [560ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [570ms] refreshing cache from master, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [571ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [575ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [576ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [578ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [610ms] refreshing cache from master, [610ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [611ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [617ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [618ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [690ms] refreshing cache from master, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [692ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [693ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [698ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [699ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [770ms] refreshing cache from master, [771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [774ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [775ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [777ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [778ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [890ms] refreshing cache from master, [891ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [895ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [896ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [897ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [898ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1270ms] refreshing cache from master, [1270ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1271ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1272ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1274ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1275ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1276ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1277ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1790ms] refreshing cache from master, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1794ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1796ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1797ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1797ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3110ms] refreshing cache from master, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [3112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3116ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [3117ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [5930ms] refreshing cache from master, [5930ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [5931ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [5932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [5935ms] too many traces: truncated at 100 traces, deferred=Deferred@334932688(state=PENDING, result=null, callback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156), errback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.102.125:46499
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29766), Traces: [0ms] refreshing cache from master, [47ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [130ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [132ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [195ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [198ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [502ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [507ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [510ms] refreshing cache from master, [511ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [513ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [515ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [518ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [520ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [522ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [523ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [530ms] refreshing cache from master, [531ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [532ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [538ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [539ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [540ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [541ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [550ms] refreshing cache from master, [550ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [551ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [552ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [555ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [557ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [559ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [560ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [570ms] refreshing cache from master, [570ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [571ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [572ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [575ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [576ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [578ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [610ms] refreshing cache from master, [610ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [611ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [612ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [615ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [617ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [618ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [690ms] refreshing cache from master, [691ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [692ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [693ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [696ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [698ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [699ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [770ms] refreshing cache from master, [771ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [772ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [773ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [774ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [775ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [777ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [778ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [890ms] refreshing cache from master, [891ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [892ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [895ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [896ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [897ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [898ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1270ms] refreshing cache from master, [1270ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1271ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1272ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1274ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1275ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1276ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1277ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1790ms] refreshing cache from master, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [1791ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [1792ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [1794ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [1796ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [1797ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [1797ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3110ms] refreshing cache from master, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [3111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [3112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.126:36543: Network error: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543, [3115ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.125:46499: Network error: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [3116ms] Sub RPC ConnectToMaster: received response from server master-127.4.102.124:38493: OK, [3117ms] delaying RPC due to: Service unavailable: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499, [5930ms] refreshing cache from master, [5930ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.125:46499, [5931ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.124:38493, [5932ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.102.126:36543, [5935ms] too many traces: truncated at 100 traces, deferred=Deferred@334932688(state=PENDING, result=null, callback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156), errback=(continuation of Deferred@1198981621 after retry RPC after error@938042170) -> (continuation of Deferred@2081068679 after retry RPC after error@1510740988) -> (continuation of Deferred@621533665 after retry RPC after error@1502056137) -> (continuation of Deferred@1896973776 after retry RPC after error@1262025725) -> (continuation of Deferred@805824426 after retry RPC after error@1427359397) -> (continuation of Deferred@676460242 after retry RPC after error@607782710) -> (continuation of Deferred@730832556 after retry RPC after error@1617056662) -> (continuation of Deferred@1193463342 after retry RPC after error@861237835) -> (continuation of Deferred@1685647475 after retry RPC after error@376758665) -> (continuation of Deferred@1198923479 after retry RPC after error@1596582744) -> (continuation of Deferred@2087809958 after retry RPC after error@1295564040) -> (continuation of Deferred@986677982 after retry RPC after error@715967988) -> (continuation of Deferred@1563916100 after retry RPC after error@1441743127) -> (continuation of Deferred@334557190 after retry RPC after error@1610570977) -> (continuation of Deferred@957652237 after retry RPC after error@375166145) -> (continuation of Deferred@45622988 after retry RPC after error@663058990) -> (continuation of Deferred@979658534 after retry RPC after error@1215043127) -> (continuation of Deferred@195155888 after retry RPC after error@125405408) -> (continuation of Deferred@515740286 after retry RPC after error@910746160) -> (continuation of Deferred@860991262 after retry RPC after error@2015633180) -> (continuation of Deferred@184048644 after retry RPC after error@2077259273) -> (continuation of Deferred@1162288946 after retry RPC after error@499332217) -> (continuation of Deferred@1435718585 after retry RPC after error@764086076) -> (continuation of Deferred@1346908976 after retry RPC after error@1412212156)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.102.125:46499,127.4.102.124:38493,127.4.102.126:36543) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.126:36543(127.4.102.126:36543): Connection refused: /127.4.102.126:36543,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.102.125:46499(127.4.102.125:46499): Connection refused: /127.4.102.125:46499
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.102.125:46499
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)