20:17:28.291 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:17:28.308 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskJJ3Nd5/test-tmp
20:17:28.311 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:17:28.315 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskJJ3Nd5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:17:28.558 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:17:28.837 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:17:28.838 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:17:28.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:28.844146 22148 mini_oidc.cc:152] Starting JWKS server
20:17:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:28.852712 22148 webserver.cc:460] Webserver started at http://127.0.0.1:42283/ using document root <none> and password file <none>
20:17:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:28.853422 22148 mini_oidc.cc:189] Starting OIDC Discovery server
20:17:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:28.858326 22148 webserver.cc:460] Webserver started at http://127.0.0.1:35035/ using document root <none> and password file <none>
20:17:28.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:17:28Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:17:28.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:17:28Z Disabled control of system clock
20:17:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:28.936415 22148 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/wal
20:17:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data
20:17:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/logs
20:17:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data/info.pb
20:17:28.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:28.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.62:42315
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.62
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:28.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42283/jwks.json with env {}
20:17:29.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.417606 22160 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:29.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.418393 22160 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.419126 22160 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.450004 22160 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.450403 22160 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.450722 22160 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.451051 22160 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.451493 22160 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:29.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.451768 22160 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42283/jwks.json
20:17:29.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:29.486971 22160 master_runner.cc:386] Master server non-default flags:
20:17:29.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data
20:17:29.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/wal
20:17:29.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.62:42315
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42283/jwks.json
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data/info.pb
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.62
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/logs
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:29.488622 22160 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:29.490785 22160 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.529199 22166 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.529748 22169 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:29.530138 22167 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:30.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:30.852381 22168 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1324 milliseconds
20:17:30.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.852495 22160 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.853995 22160 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:30.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.857384 22160 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:30.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.858814 22160 hybrid_clock.cc:648] HybridClock initialized: now 1786047450858778 us; error 62 us; skew 500 ppm
20:17:30.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.859612 22160 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.874595 22160 webserver.cc:460] Webserver started at http://127.21.161.62:43411/ using document root <none> and password file <none>
20:17:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.875597 22160 fs_manager.cc:359] Metadata directory not provided
20:17:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.875784 22160 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.876206 22160 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.881424 22160 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data/instance:
20:17:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "238254fb493a4d3098d335053f0181d9"
20:17:30.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:30 on dist-test-slave-8n49"
20:17:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.882969 22160 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/wal/instance:
20:17:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "238254fb493a4d3098d335053f0181d9"
20:17:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:30 on dist-test-slave-8n49"
20:17:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.892935 22160 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
20:17:30.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.902518 22176 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.903905 22160 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.005s sys 0.005s
20:17:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.904297 22160 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/wal
20:17:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "238254fb493a4d3098d335053f0181d9"
20:17:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:30 on dist-test-slave-8n49"
20:17:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.904808 22160 fs_report.cc:383] FS layout report
20:17:30.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/wal
20:17:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/wal
20:17:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data/data
20:17:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:30.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:30.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:30.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.966400 22160 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.968966 22160 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:30.969508 22160 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.174553 22160 rpc_server.cc:307] RPC server started. Bound to: 127.21.161.62:42315
20:17:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.174777 22228 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.161.62:42315 every 8 connection(s)
20:17:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.179404 22160 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/data/info.pb
20:17:31.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.189186 22148 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22160
20:17:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.189802 22148 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-0/wal/instance
20:17:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.191823 22229 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:17:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.195693 22148 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/wal
20:17:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data
20:17:31.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/logs
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data/info.pb
20:17:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:31.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:31.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:31.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:31.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.61:40375
20:17:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.61
20:17:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42283/jwks.json with env {}
20:17:31.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.218596 22229 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } has no permanent_uuid. Determining permanent_uuid...
20:17:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.267378 22229 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } has no permanent_uuid. Determining permanent_uuid...
20:17:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.271180 22178 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:31.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.283160 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.61:40375: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:31.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.332933 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } attempt: 1
20:17:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.339176 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.61:40375: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.407846 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } attempt: 2
20:17:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.414521 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.61:40375: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.506136 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } attempt: 3
20:17:31.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.511973 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.61:40375: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:31.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.655401 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } attempt: 4
20:17:31.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.655545 22231 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:31.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.656518 22231 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.657296 22231 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.659989 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.61:40375: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.689021 22231 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.689288 22231 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.689523 22231 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.689750 22231 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.689930 22231 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:31.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.690070 22231 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42283/jwks.json
20:17:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.724807 22231 master_runner.cc:386] Master server non-default flags:
20:17:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:31.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data
20:17:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/wal
20:17:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:31.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.61:40375
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42283/jwks.json
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data/info.pb
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.61
20:17:31.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:31.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/logs
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.726085 22231 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.728227 22231 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:31.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.759411 22241 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:31.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:31.959434 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } attempt: 5
20:17:31.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.964187 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.61:40375: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:32.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:32.511852 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } attempt: 6
20:17:32.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:32.520339 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.61:40375: Network error: Client connection negotiation failed: client connection to 127.21.161.61:40375: connect: Connection refused (error 111)
20:17:33.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.761698 22243 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:33.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:31.761595 22240 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:33.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.060117 22242 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:17:33.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.060245 22231 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:33.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.064224 22231 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:33.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.067886 22231 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:33.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.069321 22231 hybrid_clock.cc:648] HybridClock initialized: now 1786047453069292 us; error 45 us; skew 500 ppm
20:17:33.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.070159 22231 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:33.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.081749 22231 webserver.cc:460] Webserver started at http://127.21.161.61:41283/ using document root <none> and password file <none>
20:17:33.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.082702 22231 fs_manager.cc:359] Metadata directory not provided
20:17:33.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.082887 22231 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:33.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.083281 22231 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.088927 22231 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data/instance:
20:17:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49f94c977d8b4254b9ad65d2c9153e2f"
20:17:33.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:33 on dist-test-slave-8n49"
20:17:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.090083 22231 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/wal/instance:
20:17:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49f94c977d8b4254b9ad65d2c9153e2f"
20:17:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:33 on dist-test-slave-8n49"
20:17:33.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.099534 22231 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
20:17:33.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.107969 22253 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.109306 22231 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.007s sys 0.000s
20:17:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.109596 22231 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/wal
20:17:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "49f94c977d8b4254b9ad65d2c9153e2f"
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:33 on dist-test-slave-8n49"
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.109898 22231 fs_report.cc:383] FS layout report
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/wal
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/wal
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data/data
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:33.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.168589 22231 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:33.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.171335 22231 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:33.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.172116 22231 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:33.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.307771 22231 rpc_server.cc:307] RPC server started. Bound to: 127.21.161.61:40375
20:17:33.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.307905 22307 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.161.61:40375 every 8 connection(s)
20:17:33.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.310874 22231 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/data/info.pb
20:17:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.316902 22308 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:17:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.317006 22148 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22231
20:17:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.317495 22148 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-1/wal/instance
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.320982 22148 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/wal
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:33.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/logs
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data/info.pb
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.60:36683
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.60
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:33.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:33.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42283/jwks.json with env {}
20:17:33.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.336457 22308 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } has no permanent_uuid. Determining permanent_uuid...
20:17:33.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.365316 22308 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } has no permanent_uuid. Determining permanent_uuid...
20:17:33.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.387423 22308 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } has no permanent_uuid. Determining permanent_uuid...
20:17:33.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.390269 22257 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.395319 22308 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.444891 22308 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 1
20:17:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.448803 22308 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.517356 22308 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 2
20:17:33.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.521679 22308 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.581213 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } attempt: 7
20:17:33.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.595630 22229 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } has no permanent_uuid. Determining permanent_uuid...
20:17:33.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.601997 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.613359 22308 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 3
20:17:33.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.617825 22308 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.660494 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 1
20:17:33.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.664316 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.687084 22310 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:33.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.687660 22310 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:33.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.688187 22310 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.720391 22310 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:33.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.720747 22310 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:33.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.720968 22310 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:33.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.721153 22310 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:33.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.721343 22310 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:33.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.721485 22310 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42283/jwks.json
20:17:33.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.745793 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 2
20:17:33.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.749837 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.757486 22310 master_runner.cc:386] Master server non-default flags:
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/wal
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.60:36683
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:33.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42283/jwks.json
20:17:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data/info.pb
20:17:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.60
20:17:33.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/logs
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:33.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:33.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:33.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:33.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.758966 22310 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.761101 22310 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:33.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.761348 22308 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 4
20:17:33.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.766300 22308 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.785745 22323 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.835282 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 3
20:17:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.839097 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:33.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:33.979732 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 4
20:17:33.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.984421 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:34.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:34.065845 22308 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 5
20:17:34.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:34.071228 22308 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:34.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:34.267895 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 5
20:17:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:34.272696 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:34.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:34.618654 22308 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 6
20:17:34.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:34.627269 22308 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:34.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:34.825098 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 6
20:17:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:34.831916 22229 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.21.161.60:36683: Network error: Client connection negotiation failed: client connection to 127.21.161.60:36683: connect: Connection refused (error 111)
20:17:35.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.786870 22326 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:33.787114 22324 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:35.052430 22325 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:17:35.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.052677 22310 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:35.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.056882 22310 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.060142 22310 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:35.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.061517 22310 hybrid_clock.cc:648] HybridClock initialized: now 1786047455061460 us; error 77 us; skew 500 ppm
20:17:35.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.062390 22310 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:35.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.069852 22310 webserver.cc:460] Webserver started at http://127.21.161.60:37105/ using document root <none> and password file <none>
20:17:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.070871 22310 fs_manager.cc:359] Metadata directory not provided
20:17:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.071107 22310 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:35.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.071669 22310 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.076530 22310 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data/instance:
20:17:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c37fa2f27901413fa2975af88474eb86"
20:17:35.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:35 on dist-test-slave-8n49"
20:17:35.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.077847 22310 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/wal/instance:
20:17:35.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c37fa2f27901413fa2975af88474eb86"
20:17:35.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:35 on dist-test-slave-8n49"
20:17:35.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.086516 22310 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.000s sys 0.008s
20:17:35.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.093672 22337 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.094759 22310 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
20:17:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.095129 22310 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/wal
20:17:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c37fa2f27901413fa2975af88474eb86"
20:17:35.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:35 on dist-test-slave-8n49"
20:17:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.095559 22310 fs_report.cc:383] FS layout report
20:17:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/wal
20:17:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/wal
20:17:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data/data
20:17:35.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:35.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:35.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.146660 22310 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:35.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.148780 22310 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:35.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.149549 22310 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.241501 22310 rpc_server.cc:307] RPC server started. Bound to: 127.21.161.60:36683
20:17:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.241616 22388 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.161.60:36683 every 8 connection(s)
20:17:35.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.244366 22310 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/data/info.pb
20:17:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.249006 22148 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22310
20:17:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.249481 22148 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/master-2/wal/instance
20:17:35.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.250314 22389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:17:35.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.267778 22389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } has no permanent_uuid. Determining permanent_uuid...
20:17:35.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.294546 22389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } has no permanent_uuid. Determining permanent_uuid...
20:17:35.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.309544 22389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } has no permanent_uuid. Determining permanent_uuid...
20:17:35.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.332799 22389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86: Bootstrap starting.
20:17:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.340873 22389 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86: Neither blocks nor log segments found. Creating new log.
20:17:35.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.343130 22389 log.cc:824] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86: Log is configured to *not* fsync() on all Append() calls
20:17:35.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.347409 22389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86: No bootstrap required, opened a new log
20:17:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.371012 22389 raft_consensus.cc:348] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:35.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.372048 22389 raft_consensus.cc:374] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:17:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.372350 22389 raft_consensus.cc:729] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c37fa2f27901413fa2975af88474eb86, State: Initialized, Role: FOLLOWER
20:17:35.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.373323 22389 consensus_queue.cc:260] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.377875 22399 sys_catalog.cc:455] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.378906 22399 sys_catalog.cc:458] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: This master's current role is: FOLLOWER
20:17:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.379666 22389 sys_catalog.cc:564] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: configured and running, proceeding with master startup.
20:17:35.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:35.408190 22343 tablet.cc:2369] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:17:35.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:35.410513 22412 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:35.410804 22412 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.402994 22310 master_runner.cc:186] Error getting master registration for 127.21.161.62:42315: OK, instance_id { permanent_uuid: "238254fb493a4d3098d335053f0181d9" instance_seqno: 1786047451111304 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:17:35.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.439495 22310 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:17:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.553408 22399 raft_consensus.cc:482] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:17:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.554039 22399 raft_consensus.cc:504] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:35.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.556782 22399 leader_election.cc:288] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 238254fb493a4d3098d335053f0181d9 (127.21.161.62:42315), 49f94c977d8b4254b9ad65d2c9153e2f (127.21.161.61:40375)
20:17:35.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.687674 22308 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 7
20:17:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.725486 22308 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: Bootstrap starting.
20:17:35.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.732923 22308 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: Neither blocks nor log segments found. Creating new log.
20:17:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.737112 22308 log.cc:824] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: Log is configured to *not* fsync() on all Append() calls
20:17:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.742594 22308 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: No bootstrap required, opened a new log
20:17:35.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.776857 22308 raft_consensus.cc:348] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.777817 22308 raft_consensus.cc:374] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:17:35.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.778117 22308 raft_consensus.cc:729] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 49f94c977d8b4254b9ad65d2c9153e2f, State: Initialized, Role: FOLLOWER
20:17:35.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.779138 22308 consensus_queue.cc:260] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:35.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.782205 22416 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:35.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.783239 22416 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [sys.catalog]: This master's current role is: FOLLOWER
20:17:35.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.784178 22308 sys_catalog.cc:564] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [sys.catalog]: configured and running, proceeding with master startup.
20:17:35.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.784482 22281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c37fa2f27901413fa2975af88474eb86" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" is_pre_election: true
20:17:35.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.785698 22281 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c37fa2f27901413fa2975af88474eb86 in term 0.
20:17:35.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.787370 22339 leader_election.cc:302] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 49f94c977d8b4254b9ad65d2c9153e2f, c37fa2f27901413fa2975af88474eb86; no voters:
20:17:35.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.788393 22399 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:17:35.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.788790 22399 raft_consensus.cc:482] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:17:35.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.789140 22399 raft_consensus.cc:3037] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 0 FOLLOWER]: Advancing to term 1
20:17:35.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.795763 22399 raft_consensus.cc:504] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:35.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.800659 22399 leader_election.cc:288] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [CANDIDATE]: Term 1 election: Requested vote from peers 238254fb493a4d3098d335053f0181d9 (127.21.161.62:42315), 49f94c977d8b4254b9ad65d2c9153e2f (127.21.161.61:40375)
20:17:35.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.801841 22281 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c37fa2f27901413fa2975af88474eb86" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "49f94c977d8b4254b9ad65d2c9153e2f"
20:17:35.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.802567 22281 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [term 0 FOLLOWER]: Advancing to term 1
20:17:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.811023 22281 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c37fa2f27901413fa2975af88474eb86 in term 1.
20:17:35.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.812919 22339 leader_election.cc:302] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 49f94c977d8b4254b9ad65d2c9153e2f, c37fa2f27901413fa2975af88474eb86; no voters:
20:17:35.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.813673 22399 raft_consensus.cc:2781] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 1 FOLLOWER]: Leader election won for term 1
20:17:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.819598 22399 raft_consensus.cc:686] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [term 1 LEADER]: Becoming Leader. State: Replica: c37fa2f27901413fa2975af88474eb86, State: Running, Role: LEADER
20:17:35.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.821157 22399 consensus_queue.cc:237] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.845387 22422 sys_catalog.cc:455] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c37fa2f27901413fa2975af88474eb86. Latest consensus state: current_term: 1 leader_uuid: "c37fa2f27901413fa2975af88474eb86" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:35.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.846498 22422 sys_catalog.cc:458] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: This master's current role is: LEADER
20:17:35.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.857954 22428 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:17:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.865347 22229 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } attempt: 7
20:17:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.866922 22428 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:17:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:35.901753 22430 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:35.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:35.902143 22430 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.918291 22281 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [term 1 FOLLOWER]: Refusing update from remote peer c37fa2f27901413fa2975af88474eb86: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:17:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.863706 22231 master_runner.cc:186] Error getting master registration for 127.21.161.62:42315: OK, instance_id { permanent_uuid: "238254fb493a4d3098d335053f0181d9" instance_seqno: 1786047451111304 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:17:35.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.919571 22231 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:17:35.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.921085 22422 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [LEADER]: Connected to new peer: Peer: permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:17:35.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:35.933553 22259 tablet.cc:2369] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:17:35.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.944867 22229 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9: Bootstrap starting.
20:17:35.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.957320 22229 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9: Neither blocks nor log segments found. Creating new log.
20:17:35.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.963084 22229 log.cc:824] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9: Log is configured to *not* fsync() on all Append() calls
20:17:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.963400 22416 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [sys.catalog]: SysCatalogTable state changed. Reason: New leader c37fa2f27901413fa2975af88474eb86. Latest consensus state: current_term: 1 leader_uuid: "c37fa2f27901413fa2975af88474eb86" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:35.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.964229 22416 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [sys.catalog]: This master's current role is: FOLLOWER
20:17:35.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.981470 22433 mvcc.cc:205] Tried to move back new op lower bound from 7315650379413528576 to 7315650379054366720. Current Snapshot: MvccSnapshot[applied={T|T < 7315650379413528576}]
20:17:35.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.985003 22229 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9: No bootstrap required, opened a new log
20:17:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.986168 22434 mvcc.cc:205] Tried to move back new op lower bound from 7315650379413528576 to 7315650379054366720. Current Snapshot: MvccSnapshot[applied={T|T < 7315650379413528576}]
20:17:35.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.997761 22422 sys_catalog.cc:455] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c37fa2f27901413fa2975af88474eb86" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:35.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.998680 22422 sys_catalog.cc:458] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: This master's current role is: LEADER
20:17:36.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:35.999972 22428 catalog_manager.cc:1347] Generated new cluster ID: 06cf7732aea7485eb983c576e0d9caff
20:17:36.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.000314 22428 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:17:36.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.005683 22416 sys_catalog.cc:455] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c37fa2f27901413fa2975af88474eb86" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:36.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.006449 22416 sys_catalog.cc:458] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f [sys.catalog]: This master's current role is: FOLLOWER
20:17:36.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.042363 22428 catalog_manager.cc:1370] Generated new certificate authority record
20:17:36.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.045159 22428 catalog_manager.cc:1504] Loading token signing keys...
20:17:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.057583 22229 raft_consensus.cc:348] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:36.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.058527 22229 raft_consensus.cc:374] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:17:36.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.058811 22229 raft_consensus.cc:729] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 238254fb493a4d3098d335053f0181d9, State: Initialized, Role: FOLLOWER
20:17:36.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.059777 22229 consensus_queue.cc:260] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } }
20:17:36.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.073710 22229 sys_catalog.cc:564] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [sys.catalog]: configured and running, proceeding with master startup.
20:17:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.074456 22203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c37fa2f27901413fa2975af88474eb86" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "238254fb493a4d3098d335053f0181d9"
20:17:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.075235 22203 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [term 0 FOLLOWER]: Advancing to term 1
20:17:36.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.077503 22204 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c37fa2f27901413fa2975af88474eb86" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "238254fb493a4d3098d335053f0181d9" is_pre_election: true
20:17:36.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.090687 22203 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c37fa2f27901413fa2975af88474eb86 in term 1.
20:17:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.093318 22428 catalog_manager.cc:5868] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86: Generated new TSK 0
20:17:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.094458 22428 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:17:36.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.095885 22439 sys_catalog.cc:455] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.096622 22439 sys_catalog.cc:458] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [sys.catalog]: This master's current role is: FOLLOWER
20:17:36.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.099366 22202 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [term 1 FOLLOWER]: Refusing update from remote peer c37fa2f27901413fa2975af88474eb86: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:17:36.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.105789 22422 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [LEADER]: Connected to new peer: Peer: permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.137980 22148 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/wal
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:36.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/logs
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data/info.pb
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.1:0
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.161.1
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.1
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:17:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.214900 22439 sys_catalog.cc:455] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c37fa2f27901413fa2975af88474eb86. Latest consensus state: current_term: 1 leader_uuid: "c37fa2f27901413fa2975af88474eb86" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:36.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.215713 22439 sys_catalog.cc:458] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [sys.catalog]: This master's current role is: FOLLOWER
20:17:36.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.215529 22448 sys_catalog.cc:455] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c37fa2f27901413fa2975af88474eb86" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:36.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.216590 22448 sys_catalog.cc:458] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9 [sys.catalog]: This master's current role is: FOLLOWER
20:17:36.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.255535 22458 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:17:36.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.500072 22422 sys_catalog.cc:455] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c37fa2f27901413fa2975af88474eb86" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "238254fb493a4d3098d335053f0181d9" member_type: VOTER last_known_addr { host: "127.21.161.62" port: 42315 } } peers { permanent_uuid: "49f94c977d8b4254b9ad65d2c9153e2f" member_type: VOTER last_known_addr { host: "127.21.161.61" port: 40375 } } peers { permanent_uuid: "c37fa2f27901413fa2975af88474eb86" member_type: VOTER last_known_addr { host: "127.21.161.60" port: 36683 } } }
20:17:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.500923 22422 sys_catalog.cc:458] T 00000000000000000000000000000000 P c37fa2f27901413fa2975af88474eb86 [sys.catalog]: This master's current role is: LEADER
20:17:36.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.634758 22446 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:36.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.635326 22446 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:36.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.635980 22446 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:36.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.670100 22446 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:36.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.671060 22446 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.161.1
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.706566 22446 tablet_server_runner.cc:78] Tablet server non-default flags:
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/wal
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.1:0
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data/info.pb
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.1
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.161.1
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/logs
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:36.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.708130 22446 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:36.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.709933 22446 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:36.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.735811 22465 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:36.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.911013 22430 catalog_manager.cc:1259] Loaded cluster ID: 06cf7732aea7485eb983c576e0d9caff
20:17:36.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.911959 22430 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: loading cluster ID for follower catalog manager: success
20:17:36.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.941639 22430 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: acquiring CA information for follower catalog manager: success
20:17:36.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.952044 22430 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 49f94c977d8b4254b9ad65d2c9153e2f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:17:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:37.295897 22458 catalog_manager.cc:1259] Loaded cluster ID: 06cf7732aea7485eb983c576e0d9caff
20:17:37.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:37.318697 22458 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9: loading cluster ID for follower catalog manager: success
20:17:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:37.404366 22458 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9: acquiring CA information for follower catalog manager: success
20:17:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:37.421653 22458 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 238254fb493a4d3098d335053f0181d9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:17:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:36.740398 22446 server_base.cc:1034] running on GCE node
20:17:38.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.740942 22468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:36.739341 22464 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.344152 22446 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:38.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.349004 22446 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:38.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.350659 22446 hybrid_clock.cc:648] HybridClock initialized: now 1786047458350587 us; error 97 us; skew 500 ppm
20:17:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.351737 22446 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.360056 22446 webserver.cc:460] Webserver started at http://127.21.161.1:40461/ using document root <none> and password file <none>
20:17:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.362036 22446 fs_manager.cc:359] Metadata directory not provided
20:17:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.362350 22446 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.363054 22446 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.371173 22446 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data/instance:
20:17:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d671dcb6e45b4663862a7df868b1c949"
20:17:38.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:38 on dist-test-slave-8n49"
20:17:38.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.372859 22446 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/wal/instance:
20:17:38.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d671dcb6e45b4663862a7df868b1c949"
20:17:38.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:38 on dist-test-slave-8n49"
20:17:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.383911 22446 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.005s sys 0.004s
20:17:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.391932 22475 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.393464 22446 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.003s
20:17:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.394033 22446 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/wal
20:17:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d671dcb6e45b4663862a7df868b1c949"
20:17:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:38 on dist-test-slave-8n49"
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.394454 22446 fs_report.cc:383] FS layout report
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/wal
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/wal
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data/data
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.455098 22446 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:38.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.457027 22446 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:38.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.457589 22446 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:38.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.461144 22446 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:17:38.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.467242 22446 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:17:38.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.467525 22446 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:17:38.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.467828 22446 ts_tablet_manager.cc:610] Registered 0 tablets
20:17:38.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.468035 22446 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:17:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.682408 22446 rpc_server.cc:307] RPC server started. Bound to: 127.21.161.1:35953
20:17:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.682523 22590 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.161.1:35953 every 8 connection(s)
20:17:38.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.687386 22446 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/data/info.pb
20:17:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.689296 22148 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22446
20:17:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.689724 22148 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-0/wal/instance
20:17:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.706211 22148 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/wal
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/logs
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data/info.pb
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:17:38.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.2:0
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.161.2
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.2
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:38.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:17:38.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.820472 22593 heartbeater.cc:347] Connected to a master server at 127.21.161.61:40375
20:17:38.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.821036 22593 heartbeater.cc:464] Registering TS with master...
20:17:38.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.822491 22593 heartbeater.cc:511] Master 127.21.161.61:40375 requested a full tablet report, sending...
20:17:38.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.826160 22271 ts_manager.cc:194] Registered new tserver with Master: d671dcb6e45b4663862a7df868b1c949 (127.21.161.1:35953)
20:17:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.830940 22591 heartbeater.cc:347] Connected to a master server at 127.21.161.60:36683
20:17:38.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.831302 22591 heartbeater.cc:464] Registering TS with master...
20:17:38.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.832836 22591 heartbeater.cc:511] Master 127.21.161.60:36683 requested a full tablet report, sending...
20:17:38.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.835408 22354 ts_manager.cc:194] Registered new tserver with Master: d671dcb6e45b4663862a7df868b1c949 (127.21.161.1:35953)
20:17:38.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.838244 22354 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.161.1:42643
20:17:38.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.847503 22596 heartbeater.cc:347] Connected to a master server at 127.21.161.62:42315
20:17:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.847862 22596 heartbeater.cc:464] Registering TS with master...
20:17:38.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.848610 22596 heartbeater.cc:511] Master 127.21.161.62:42315 requested a full tablet report, sending...
20:17:38.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:38.851673 22194 ts_manager.cc:194] Registered new tserver with Master: d671dcb6e45b4663862a7df868b1c949 (127.21.161.1:35953)
20:17:39.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.141234 22595 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.141800 22595 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.142446 22595 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.174911 22595 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:39.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.175931 22595 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.161.2
20:17:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:39.213171 22595 tablet_server_runner.cc:78] Tablet server non-default flags:
20:17:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data
20:17:39.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/wal
20:17:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.2:0
20:17:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data/info.pb
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.2
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.161.2
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/logs
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:39.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:39.214601 22595 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:39.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:39.216652 22595 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:39.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.291531 22609 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:39.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:39.843266 22591 heartbeater.cc:503] Master 127.21.161.60:36683 was elected leader, sending a full tablet report...
20:17:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.291888 22612 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:40.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:39.296030 22610 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:40.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.746995 22595 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:40.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:40.746887 22611 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1463 milliseconds
20:17:40.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.752661 22595 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:40.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.756057 22595 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:40.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.757479 22595 hybrid_clock.cc:648] HybridClock initialized: now 1786047460757455 us; error 41 us; skew 500 ppm
20:17:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.758263 22595 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:40.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.774304 22595 webserver.cc:460] Webserver started at http://127.21.161.2:41175/ using document root <none> and password file <none>
20:17:40.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.775993 22595 fs_manager.cc:359] Metadata directory not provided
20:17:40.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.776587 22595 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:40.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.777415 22595 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.784452 22595 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data/instance:
20:17:40.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7834acee8c1948d88317dfc4d030190c"
20:17:40.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:40 on dist-test-slave-8n49"
20:17:40.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.785755 22595 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/wal/instance:
20:17:40.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7834acee8c1948d88317dfc4d030190c"
20:17:40.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:40 on dist-test-slave-8n49"
20:17:40.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.796036 22595 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.005s sys 0.005s
20:17:40.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.806672 22620 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:40.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.807829 22595 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.002s sys 0.008s
20:17:40.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.808148 22595 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/wal
20:17:40.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7834acee8c1948d88317dfc4d030190c"
20:17:40.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:40 on dist-test-slave-8n49"
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.808445 22595 fs_report.cc:383] FS layout report
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/wal
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/wal
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data/data
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.871615 22595 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:40.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.874521 22595 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:40.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.875350 22595 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:40.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.882337 22595 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:17:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.890792 22595 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:17:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.891028 22595 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:17:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.891428 22595 ts_tablet_manager.cc:610] Registered 0 tablets
20:17:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:40.891615 22595 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:17:41.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.430388 22595 rpc_server.cc:307] RPC server started. Bound to: 127.21.161.2:46415
20:17:41.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.430627 22734 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.161.2:46415 every 8 connection(s)
20:17:41.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.433781 22595 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/data/info.pb
20:17:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.439590 22148 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22595
20:17:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.440774 22148 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-1/wal/instance
20:17:41.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.511256 22148 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/wal
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:41.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/logs
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data/info.pb
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.3:0
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.161.3
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.3
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:41.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:41.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:17:41.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.593384 22736 heartbeater.cc:347] Connected to a master server at 127.21.161.61:40375
20:17:41.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.594022 22736 heartbeater.cc:464] Registering TS with master...
20:17:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.595881 22736 heartbeater.cc:511] Master 127.21.161.61:40375 requested a full tablet report, sending...
20:17:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.600962 22271 ts_manager.cc:194] Registered new tserver with Master: 7834acee8c1948d88317dfc4d030190c (127.21.161.2:46415)
20:17:41.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.605324 22735 heartbeater.cc:347] Connected to a master server at 127.21.161.60:36683
20:17:41.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.605794 22735 heartbeater.cc:464] Registering TS with master...
20:17:41.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.607112 22735 heartbeater.cc:511] Master 127.21.161.60:36683 requested a full tablet report, sending...
20:17:41.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.611056 22354 ts_manager.cc:194] Registered new tserver with Master: 7834acee8c1948d88317dfc4d030190c (127.21.161.2:46415)
20:17:41.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.611713 22739 heartbeater.cc:347] Connected to a master server at 127.21.161.62:42315
20:17:41.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.612192 22739 heartbeater.cc:464] Registering TS with master...
20:17:41.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.613518 22739 heartbeater.cc:511] Master 127.21.161.62:42315 requested a full tablet report, sending...
20:17:41.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.614920 22354 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.161.2:37595
20:17:41.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:41.618063 22194 ts_manager.cc:194] Registered new tserver with Master: 7834acee8c1948d88317dfc4d030190c (127.21.161.2:46415)
20:17:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:41.963774 22745 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:41.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:41.964340 22745 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:41.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:41.964987 22745 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:41.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:41.997104 22745 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:41.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:41.998092 22745 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.161.3
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:42.037657 22745 tablet_server_runner.cc:78] Tablet server non-default flags:
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.161.20:43303
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/wal
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.161.3:0
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data/info.pb
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.161.3
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.161.62:42315,127.21.161.61:40375,127.21.161.60:36683
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.161.3
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/logs
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:42.039405 22745 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:42.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:42.041352 22745 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:42.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:42.068605 22753 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:42.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:42.624009 22735 heartbeater.cc:503] Master 127.21.161.60:36683 was elected leader, sending a full tablet report...
20:17:42.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:41.552634 22157 outbound_call.cc:322] RPC callback for RPC call kudu.master.MasterService.ConnectToMaster -> {remote=127.21.161.60:36683, user_credentials={real_user=slave}} blocked reactor thread for 45024.5us
20:17:43.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:42.072832 22745 server_base.cc:1034] running on GCE node
20:17:43.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:42.071188 22754 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:43.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:42.074467 22756 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:43.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.496754 22745 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:43.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.500542 22745 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:43.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.501991 22745 hybrid_clock.cc:648] HybridClock initialized: now 1786047463501946 us; error 57 us; skew 500 ppm
20:17:43.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.502847 22745 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:43.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.514308 22745 webserver.cc:460] Webserver started at http://127.21.161.3:42707/ using document root <none> and password file <none>
20:17:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.515216 22745 fs_manager.cc:359] Metadata directory not provided
20:17:43.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.515458 22745 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.516044 22745 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.522692 22745 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data/instance:
20:17:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b1e96d6d0dd477a97b4d7275a6f2637"
20:17:43.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:43 on dist-test-slave-8n49"
20:17:43.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.523803 22745 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/wal/instance:
20:17:43.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b1e96d6d0dd477a97b4d7275a6f2637"
20:17:43.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:43 on dist-test-slave-8n49"
20:17:43.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.534065 22745 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.011s sys 0.001s
20:17:43.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.542443 22763 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:43.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.543659 22745 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.543984 22745 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/wal
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b1e96d6d0dd477a97b4d7275a6f2637"
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:43 on dist-test-slave-8n49"
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.544320 22745 fs_report.cc:383] FS layout report
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/wal
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/wal
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data/data
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:43.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:43.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.631510 22745 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.634105 22745 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.634977 22745 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:43.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.640448 22745 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:17:43.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.647791 22745 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:17:43.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.647998 22745 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:17:43.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.648228 22745 ts_tablet_manager.cc:610] Registered 0 tablets
20:17:43.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.648363 22745 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:17:43.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.896732 22745 rpc_server.cc:307] RPC server started. Bound to: 127.21.161.3:33159
20:17:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.897012 22878 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.21.161.3:33159 every 8 connection(s)
20:17:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.899449 22745 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/data/info.pb
20:17:43.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.903990 22148 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22745
20:17:43.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.904458 22148 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster8379798486492425627/ts-2/wal/instance
20:17:43.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.990514 22879 heartbeater.cc:347] Connected to a master server at 127.21.161.60:36683
20:17:43.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.991140 22879 heartbeater.cc:464] Registering TS with master...
20:17:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.993141 22879 heartbeater.cc:511] Master 127.21.161.60:36683 requested a full tablet report, sending...
20:17:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.994573 22882 heartbeater.cc:347] Connected to a master server at 127.21.161.62:42315
20:17:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.994982 22880 heartbeater.cc:347] Connected to a master server at 127.21.161.61:40375
20:17:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.995157 22882 heartbeater.cc:464] Registering TS with master...
20:17:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.995537 22880 heartbeater.cc:464] Registering TS with master...
20:17:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.997336 22880 heartbeater.cc:511] Master 127.21.161.61:40375 requested a full tablet report, sending...
20:17:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.997473 22882 heartbeater.cc:511] Master 127.21.161.62:42315 requested a full tablet report, sending...
20:17:43.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:43.998340 22353 ts_manager.cc:194] Registered new tserver with Master: 6b1e96d6d0dd477a97b4d7275a6f2637 (127.21.161.3:33159)
20:17:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:44.001307 22194 ts_manager.cc:194] Registered new tserver with Master: 6b1e96d6d0dd477a97b4d7275a6f2637 (127.21.161.3:33159)
20:17:44.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:44.001945 22353 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.21.161.3:51543
20:17:44.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:44.006372 22271 ts_manager.cc:194] Registered new tserver with Master: 6b1e96d6d0dd477a97b4d7275a6f2637 (127.21.161.3:33159)
20:17:44.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:44.027045 22148 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:17:44.049 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:17:44.050 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:17:44.066 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.161.62"
port: 42315
}
bound_http_address {
host: "127.21.161.62"
port: 43411
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.161.61"
port: 40375
}
bound_http_address {
host: "127.21.161.61"
port: 41283
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.161.60"
port: 36683
}
bound_http_address {
host: "127.21.161.60"
port: 37105
}
}
}
20:17:44.073 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:17:44.077 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.161.1"
port: 35953
}
bound_http_address {
host: "127.21.161.1"
port: 40461
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.161.2"
port: 46415
}
bound_http_address {
host: "127.21.161.2"
port: 41175
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.161.3"
port: 33159
}
bound_http_address {
host: "127.21.161.3"
port: 42707
}
}
}
20:17:44.078 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:17:44.335 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:17:44.507 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskJJ3Nd5/test-tmp
20:17:44.508 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:17:44.509 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskJJ3Nd5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:17:44.512 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:17:45.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:45.008574 22879 heartbeater.cc:503] Master 127.21.161.60:36683 was elected leader, sending a full tablet report...
20:17:45.101 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:45.101 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:17:45.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:45.104998 22890 mini_oidc.cc:152] Starting JWKS server
20:17:45.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:45.113665 22890 webserver.cc:460] Webserver started at http://127.0.0.1:38095/ using document root <none> and password file <none>
20:17:45.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:45.114352 22890 mini_oidc.cc:189] Starting OIDC Discovery server
20:17:45.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:45.120249 22890 webserver.cc:460] Webserver started at http://127.0.0.1:44049/ using document root <none> and password file <none>
20:17:45.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:45.145481 22875 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 22748
20:17:45.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:45.147081 22875 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff40 after lost signal to thread 22878
20:17:45.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:17:45.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/principal' for realm 'KRBTEST.COM',
20:17:45.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:17:45.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:45 dist-test-slave-8n49 krb5kdc[22903](info): setting up network...
20:17:45.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:17:45.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:45 dist-test-slave-8n49 krb5kdc[22903](info): set up 2 sockets
20:17:45.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:45 dist-test-slave-8n49 krb5kdc[22903](info): commencing operation
20:17:45.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:17:46.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:46.420888 22890 mini_kdc.cc:121] Time spent starting KDC: real 1.256s user 0.011s sys 0.023s
20:17:46.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:17:46.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:17:46.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:17:46.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:17:46.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:17:46.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:17:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:17:46.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:17:46.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:17:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:17:46.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/test-user.keytab.
20:17:46.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/test-user.keytab.
20:17:46.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:46 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047466, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:46.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:17:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:17:46Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:17:46.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:17:46Z Disabled control of system clock
20:17:46.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.90.190@KRBTEST.COM; defaulting to no policy
20:17:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:17:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.90.190@KRBTEST.COM" created.
20:17:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:17:46.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.90.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.190.keytab.
20:17:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.90.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.190.keytab.
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:46.661231 22890 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/logs
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/info.pb
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.190:36727
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.190
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.190.keytab
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.190
20:17:46.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:46.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:46.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:17:47.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.094201 22917 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:47.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.094811 22917 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.095436 22917 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:47.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.134460 22917 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:17:47.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.134692 22917 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.134963 22917 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.135159 22917 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:47.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.135533 22917 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.135890 22917 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:47.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.136125 22917 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38095/jwks.json
20:17:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:47.189040 22917 master_runner.cc:386] Master server non-default flags:
20:17:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.190.keytab
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.190
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.190:36727
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/info.pb
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.190
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/logs
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:47.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:47.190589 22917 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:47.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:47.191880 22917 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:47.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.226992 22924 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:48.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.229015 22926 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:48.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:47.230170 22923 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:48.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:48.677304 22925 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1449 milliseconds
20:17:48.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.677448 22917 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:48.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.678774 22917 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.682745 22917 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:48.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.684191 22917 hybrid_clock.cc:648] HybridClock initialized: now 1786047468684147 us; error 60 us; skew 500 ppm
20:17:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.685345 22917 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:48 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047468, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:48.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.697722 22917 init.cc:377] Logged in from keytab as kudu/127.22.90.190@KRBTEST.COM (short username kudu)
20:17:48.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.706354 22917 webserver.cc:460] Webserver started at http://127.22.90.190:33245/ using document root <none> and password file <none>
20:17:48.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.707237 22917 fs_manager.cc:359] Metadata directory not provided
20:17:48.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.707419 22917 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:48.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.707803 22917 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:48.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.713054 22917 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/instance:
20:17:48.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d328412ce517409587d18c320e958811"
20:17:48.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:48 on dist-test-slave-8n49"
20:17:48.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.714147 22917 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal/instance:
20:17:48.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d328412ce517409587d18c320e958811"
20:17:48.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:48 on dist-test-slave-8n49"
20:17:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.724064 22917 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
20:17:48.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.732990 22935 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:48.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.734293 22917 fs_manager.cc:725] Time spent opening block manager: real 0.007s user 0.008s sys 0.000s
20:17:48.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.734619 22917 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:48.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d328412ce517409587d18c320e958811"
20:17:48.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:48 on dist-test-slave-8n49"
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.734930 22917 fs_report.cc:383] FS layout report
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/data
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.801481 22917 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:48.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.808334 22917 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:48.808990 22917 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:49.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.000458 22917 rpc_server.cc:307] RPC server started. Bound to: 127.22.90.190:36727
20:17:49.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.000798 22989 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.90.190:36727 every 8 connection(s)
20:17:49.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.003818 22917 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/info.pb
20:17:49.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.004910 22890 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22917
20:17:49.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.005306 22890 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal/instance
20:17:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.017025 22990 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:17:49.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.90.189@KRBTEST.COM; defaulting to no policy
20:17:49.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:17:49.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.90.189@KRBTEST.COM" created.
20:17:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.046222 22990 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } has no permanent_uuid. Determining permanent_uuid...
20:17:49.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:17:49.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.90.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.189.keytab.
20:17:49.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.90.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.189.keytab.
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.072789 22890 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/logs
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/info.pb
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:49.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.189:36071
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.189
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.189.keytab
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.189
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:17:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:49 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047468, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.190@KRBTEST.COM for kudu/127.22.90.190@KRBTEST.COM
20:17:49.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.123275 22990 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } has no permanent_uuid. Determining permanent_uuid...
20:17:49.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.127306 22938 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.135783 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.189:36071: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:49.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.185299 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } attempt: 1
20:17:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.189970 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.189:36071: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:49.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.258396 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } attempt: 2
20:17:49.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.265414 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.189:36071: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.356794 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } attempt: 3
20:17:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.360718 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.189:36071: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:49.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.504164 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } attempt: 4
20:17:49.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.508805 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.189:36071: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:49.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.512302 22996 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.512948 22996 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:49.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.513557 22996 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:49.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.546401 22996 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:17:49.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.546638 22996 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.546895 22996 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.547091 22996 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.547272 22996 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.547470 22996 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:49.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.547607 22996 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38095/jwks.json
20:17:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.585290 22996 master_runner.cc:386] Master server non-default flags:
20:17:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:49.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.189.keytab
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.189
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.189:36071
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/info.pb
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.189
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/logs
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.586439 22996 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:49.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.587469 22996 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:49.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.614102 23008 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:49.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:49.808256 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } attempt: 5
20:17:49.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.814391 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.189:36071: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:50.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.361898 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } attempt: 6
20:17:50.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:50.370025 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.189:36071: Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:17:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.614115 23006 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:49.620723 23005 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:50.954515 23007 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:17:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.954661 22996 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:50.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.958907 22996 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:50.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.962146 22996 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:50.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.963562 22996 hybrid_clock.cc:648] HybridClock initialized: now 1786047470963480 us; error 54 us; skew 500 ppm
20:17:50.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.964599 22996 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:50.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:50 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047470, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:50.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.973278 22996 init.cc:377] Logged in from keytab as kudu/127.22.90.189@KRBTEST.COM (short username kudu)
20:17:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.984742 22996 webserver.cc:460] Webserver started at http://127.22.90.189:38595/ using document root <none> and password file <none>
20:17:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.985860 22996 fs_manager.cc:359] Metadata directory not provided
20:17:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.986101 22996 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:50.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.986687 22996 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.993557 22996 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/instance:
20:17:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6"
20:17:50.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:50 on dist-test-slave-8n49"
20:17:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:50.995213 22996 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal/instance:
20:17:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6"
20:17:50.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:50 on dist-test-slave-8n49"
20:17:51.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.007830 22996 fs_manager.cc:691] Time spent creating directory manager: real 0.012s user 0.004s sys 0.008s
20:17:51.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.018589 23018 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:51.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.019974 22996 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.004s sys 0.005s
20:17:51.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.020296 22996 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:51.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6"
20:17:51.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:50 on dist-test-slave-8n49"
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.020749 22996 fs_report.cc:383] FS layout report
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/data
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:51.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:51.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.082697 22996 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.090611 22996 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:51.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.091518 22996 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:51.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.352465 22996 rpc_server.cc:307] RPC server started. Bound to: 127.22.90.189:36071
20:17:51.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.352604 23071 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.90.189:36071 every 8 connection(s)
20:17:51.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.355708 22996 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/info.pb
20:17:51.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.366058 22890 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 22996
20:17:51.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.366850 22890 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal/instance
20:17:51.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.367655 23072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:17:51.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.90.188@KRBTEST.COM; defaulting to no policy
20:17:51.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:17:51.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.90.188@KRBTEST.COM" created.
20:17:51.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.398617 23072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } has no permanent_uuid. Determining permanent_uuid...
20:17:51.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.430419 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } attempt: 7
20:17:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:17:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.90.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.188.keytab.
20:17:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.90.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.188.keytab.
20:17:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.455174 22890 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data
20:17:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:51.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/logs
20:17:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/info.pb
20:17:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:51.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.188:35103
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.188
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.188.keytab
20:17:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.188
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:51.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:17:51.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:51 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047470, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.189@KRBTEST.COM for kudu/127.22.90.190@KRBTEST.COM
20:17:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:51 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047468, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.190@KRBTEST.COM for kudu/127.22.90.189@KRBTEST.COM
20:17:51.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.483620 23072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } has no permanent_uuid. Determining permanent_uuid...
20:17:51.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:51 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047470, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.189@KRBTEST.COM for kudu/127.22.90.189@KRBTEST.COM
20:17:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.512408 22990 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } has no permanent_uuid. Determining permanent_uuid...
20:17:51.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.532610 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.538616 23072 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } has no permanent_uuid. Determining permanent_uuid...
20:17:51.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.543188 23019 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.554448 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.591276 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 1
20:17:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.598268 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.604031 23072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 1
20:17:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.612298 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.680040 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 2
20:17:51.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.681190 23072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 2
20:17:51.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.689895 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.691135 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.776602 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 3
20:17:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.781317 23072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 3
20:17:51.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.783844 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.788316 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.924484 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 4
20:17:51.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.930635 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:51.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:51.931993 23072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 4
20:17:51.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:51.938660 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:52.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.023216 23080 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:52.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.023759 23080 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:52.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.024329 23080 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.061144 23080 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:17:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.061374 23080 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.061645 23080 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:52.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.061851 23080 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:52.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.062031 23080 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:52.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.062229 23080 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:52.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.062366 23080 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38095/jwks.json
20:17:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:52.099596 23080 master_runner.cc:386] Master server non-default flags:
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.188.keytab
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.188
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.188:35103
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/info.pb
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.188
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/logs
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:52.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:52.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:52.100878 23080 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:52.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:52.101830 23080 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:52.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.123096 23091 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:52.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:52.214126 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 5
20:17:52.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.220780 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:52.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:52.238219 23072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 5
20:17:52.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.244289 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:52.773242 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 6
20:17:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.780380 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:52.792006 23072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 6
20:17:52.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.799176 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:53.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.123977 23092 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:53.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:52.124701 23094 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:53.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:53.563889 23093 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:17:53.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.564414 23080 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.568789 23080 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:53.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.573349 23080 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.574800 23080 hybrid_clock.cc:648] HybridClock initialized: now 1786047473574765 us; error 46 us; skew 500 ppm
20:17:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.575695 23080 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:53.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:53 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047473, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:53.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.584246 23080 init.cc:377] Logged in from keytab as kudu/127.22.90.188@KRBTEST.COM (short username kudu)
20:17:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.593152 23080 webserver.cc:460] Webserver started at http://127.22.90.188:45303/ using document root <none> and password file <none>
20:17:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.594204 23080 fs_manager.cc:359] Metadata directory not provided
20:17:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.594483 23080 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:53.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.595031 23080 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:17:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.600872 23080 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/instance:
20:17:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbb55c2886a415eb3483ad9bc885ba8"
20:17:53.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:53 on dist-test-slave-8n49"
20:17:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.602108 23080 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal/instance:
20:17:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbb55c2886a415eb3483ad9bc885ba8"
20:17:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:53 on dist-test-slave-8n49"
20:17:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.613019 23080 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.002s sys 0.009s
20:17:53.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.622886 23106 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:53.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.624229 23080 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.006s sys 0.003s
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.624653 23080 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbb55c2886a415eb3483ad9bc885ba8"
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:53 on dist-test-slave-8n49"
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.625087 23080 fs_report.cc:383] FS layout report
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/data
20:17:53.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:53.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:53.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.697386 23080 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:53.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.708284 23080 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.709002 23080 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.814052 22990 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 7
20:17:53.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:53.829720 22990 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:53.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.859843 23072 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } attempt: 7
20:17:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:53.874042 23072 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.90.188:35103: Network error: Client connection negotiation failed: client connection to 127.22.90.188:35103: connect: Connection refused (error 111)
20:17:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.904371 23080 rpc_server.cc:307] RPC server started. Bound to: 127.22.90.188:35103
20:17:53.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.904454 23162 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.90.188:35103 every 8 connection(s)
20:17:53.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.907696 23080 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/info.pb
20:17:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.915208 22890 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 23080
20:17:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.915935 22890 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal/instance
20:17:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.919039 23163 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:17:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:53.946693 23163 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } has no permanent_uuid. Determining permanent_uuid...
20:17:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:53 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047466, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.90.188@KRBTEST.COM
20:17:53.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:53 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047473, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.188@KRBTEST.COM for kudu/127.22.90.190@KRBTEST.COM
20:17:54.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.013656 23163 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } has no permanent_uuid. Determining permanent_uuid...
20:17:54.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:54 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047473, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.188@KRBTEST.COM for kudu/127.22.90.189@KRBTEST.COM
20:17:54.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:54 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047466, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.90.189@KRBTEST.COM
20:17:54.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.066973 23163 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } has no permanent_uuid. Determining permanent_uuid...
20:17:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:54 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047473, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.188@KRBTEST.COM for kudu/127.22.90.188@KRBTEST.COM
20:17:54.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:54 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047466, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.90.190@KRBTEST.COM
20:17:54.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.117900 22890 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:17:54.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.120791 23163 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Bootstrap starting.
20:17:54.131 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:54.133 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:17:54.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.136978 23163 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Neither blocks nor log segments found. Creating new log.
20:17:54.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.141538 23163 log.cc:824] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Log is configured to *not* fsync() on all Append() calls
20:17:54.148 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:17:54.149 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:17:54.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.152216 23163 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: No bootstrap required, opened a new log
20:17:54.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.90.190"
port: 36727
}
bound_http_address {
host: "127.22.90.190"
port: 33245
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.90.189"
port: 36071
}
bound_http_address {
host: "127.22.90.189"
port: 38595
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.90.188"
port: 35103
}
bound_http_address {
host: "127.22.90.188"
port: 45303
}
}
}
20:17:54.154 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:17:54.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:54.155 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:17:54.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:17:54.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.195365 23163 raft_consensus.cc:348] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:17:54.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.196529 23163 raft_consensus.cc:374] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:17:54.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.197170 23163 raft_consensus.cc:729] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfbb55c2886a415eb3483ad9bc885ba8, State: Initialized, Role: FOLLOWER
20:17:54.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.198679 23163 consensus_queue.cc:260] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:17:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:54 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047474, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:17:54.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.207825 23175 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } } }
20:17:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.209796 23175 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [sys.catalog]: This master's current role is: FOLLOWER
20:17:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.210238 23163 sys_catalog.cc:564] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [sys.catalog]: configured and running, proceeding with master startup.
20:17:54.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.216944 23113 tablet.cc:2369] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:17:54.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:54.234 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc
20:17:54.247 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:17:54.248 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc
20:17:54.251 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:17:54.259 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:17:54.261 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:17:54.262 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:17:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:54 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047474, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:17:54.327 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:54.328 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc
20:17:54.330 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:17:54.330 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.90.189:36071
20:17:54.332 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:17:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.333424 22890 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22996
20:17:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.341210 23188 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:17:54.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.341609 23188 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:17:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.300659 23080 master_runner.cc:186] Error getting master registration for 127.22.90.190:36727: OK, instance_id { permanent_uuid: "d328412ce517409587d18c320e958811" instance_seqno: 1786047468938226 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:17:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.364374 23080 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:17:54.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:54.367 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.90.190:36727
20:17:54.368 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:17:54.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.368454 22890 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22917
20:17:54.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:54.394 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.90.188:35103
20:17:54.394 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:17:54.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.394836 22890 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 23080
20:17:54.420 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:54.420 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.90.189:36071
20:17:54.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.424619 22890 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/logs
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/info.pb
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:54.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.189:36071
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.189
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38595
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.189.keytab
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.189
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:17:54.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.743916 23191 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:54.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.744626 23191 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.745374 23191 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.777798 23191 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:17:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.778088 23191 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.778467 23191 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:54.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.778796 23191 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:54.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.779104 23191 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:54.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.779410 23191 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:54.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.779644 23191 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38095/jwks.json
20:17:54.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.817365 23191 master_runner.cc:386] Master server non-default flags:
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.189.keytab
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.189
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.189:36071
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/info.pb
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.189
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38595
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/logs
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.818949 23191 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.820184 23191 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.838445 23198 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:56.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.838438 23200 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:54.839710 23191 server_base.cc:1034] running on GCE node
20:17:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:54.843418 23197 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.071244 23191 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:56.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.085134 23191 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:56.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.086752 23191 hybrid_clock.cc:648] HybridClock initialized: now 1786047476086698 us; error 51 us; skew 500 ppm
20:17:56.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.087680 23191 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:56.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:56 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047476, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:56.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.094136 23191 init.cc:377] Logged in from keytab as kudu/127.22.90.189@KRBTEST.COM (short username kudu)
20:17:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.099587 23191 webserver.cc:460] Webserver started at http://127.22.90.189:38595/ using document root <none> and password file <none>
20:17:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.100546 23191 fs_manager.cc:359] Metadata directory not provided
20:17:56.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.100831 23191 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:56.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.108793 23191 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
20:17:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.112975 23210 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.114050 23191 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:17:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.114311 23191 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6"
20:17:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:50 on dist-test-slave-8n49"
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.116706 23191 fs_report.cc:383] FS layout report
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/data
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:56.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.159304 23191 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.165584 23191 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:56.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.166152 23191 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.244997 23191 rpc_server.cc:307] RPC server started. Bound to: 127.22.90.189:36071
20:17:56.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.245127 23263 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.90.189:36071 every 8 connection(s)
20:17:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.248039 23191 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/data/info.pb
20:17:56.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.258282 22890 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 23191
20:17:56.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:56.259 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.90.190:36727
20:17:56.259 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:17:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.261003 22890 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:56.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/logs
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/info.pb
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:56.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.190:36727
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.190
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33245
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.190.keytab
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.190
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:56.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:17:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.262194 23264 sys_catalog.cc:263] Verifying existing consensus state
20:17:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 20:17:56.263109 23264 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:17:56.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.264480 23191 master.cc:537] Master@127.22.90.189:36071 shutting down...
20:17:56.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.336225 23191 master.cc:552] Master@127.22.90.189:36071 shutdown complete.
20:17:56.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:17:56.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.588181 23266 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:56.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.588934 23266 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:56.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.589840 23266 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.622931 23266 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:17:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.623152 23266 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.623409 23266 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.623605 23266 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:56.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.623790 23266 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:56.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.623981 23266 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:56.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.624118 23266 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38095/jwks.json
20:17:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.663892 23266 master_runner.cc:386] Master server non-default flags:
20:17:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.190.keytab
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.190
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.190:36727
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:56.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/info.pb
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.190
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33245
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/logs
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.665297 23266 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:56.666468 23266 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:56.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.685359 23274 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:57.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.686142 23276 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:56.686954 23273 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:57.921429 23275 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:17:57.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.921589 23266 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:17:57.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.925943 23266 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.929023 23266 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.930400 23266 hybrid_clock.cc:648] HybridClock initialized: now 1786047477930362 us; error 53 us; skew 500 ppm
20:17:57.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.931468 23266 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:57 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047477, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:57.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.937271 23266 init.cc:377] Logged in from keytab as kudu/127.22.90.190@KRBTEST.COM (short username kudu)
20:17:57.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.942549 23266 webserver.cc:460] Webserver started at http://127.22.90.190:33245/ using document root <none> and password file <none>
20:17:57.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.943482 23266 fs_manager.cc:359] Metadata directory not provided
20:17:57.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.943740 23266 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:57.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.952260 23266 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
20:17:57.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.957346 23284 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:57.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.958433 23266 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.958730 23266 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d328412ce517409587d18c320e958811"
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:48 on dist-test-slave-8n49"
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.961169 23266 fs_report.cc:383] FS layout report
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/data
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:57.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.994156 23266 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:57.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.999482 23266 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:57.999945 23266 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.073287 23266 rpc_server.cc:307] RPC server started. Bound to: 127.22.90.190:36727
20:17:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.073330 23339 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.90.190:36727 every 8 connection(s)
20:17:58.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.076270 23266 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/data/info.pb
20:17:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.080770 22890 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 23266
20:17:58.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:58.082 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.90.188:35103
20:17:58.082 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.084667 22890 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/logs
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/info.pb
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:58.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.188:35103
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.188
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45303
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.188.keytab
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.188
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:58.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:58.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:17:58.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.091406 23340 sys_catalog.cc:263] Verifying existing consensus state
20:17:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 20:17:58.092247 23340 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:17:58.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.092736 23266 master.cc:537] Master@127.22.90.190:36727 shutting down...
20:17:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.156910 23266 master.cc:552] Master@127.22.90.190:36727 shutdown complete.
20:17:58.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:17:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.417565 23342 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:17:58.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.418339 23342 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:17:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.419178 23342 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:17:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.450312 23342 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:17:58.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.450621 23342 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:17:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.451031 23342 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:17:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.451371 23342 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:17:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.451695 23342 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:17:58.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.452013 23342 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:17:58.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.452245 23342 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38095/jwks.json
20:17:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.489351 23342 master_runner.cc:386] Master server non-default flags:
20:17:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:17:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.90.148:41653
20:17:58.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.90.190:36727,127.22.90.189:36071,127.22.90.188:35103
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/krb5kdc/kudu_127.22.90.188.keytab
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.90.188
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.90.188:35103
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38095/jwks.json
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/info.pb
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.90.188
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45303
20:17:58.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/logs
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:17:58.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.490849 23342 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:17:58.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.492161 23342 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:17:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.509155 23348 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.509495 23349 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:58.511638 23342 server_base.cc:1034] running on GCE node
20:17:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:17:58.512733 23351 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:17:59.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.750164 23342 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:17:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.753013 23342 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:17:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.754390 23342 hybrid_clock.cc:648] HybridClock initialized: now 1786047479754344 us; error 55 us; skew 500 ppm
20:17:59.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.755283 23342 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:17:59.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:17:59 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047479, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:17:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.764940 23342 init.cc:377] Logged in from keytab as kudu/127.22.90.188@KRBTEST.COM (short username kudu)
20:17:59.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.770174 23342 webserver.cc:460] Webserver started at http://127.22.90.188:45303/ using document root <none> and password file <none>
20:17:59.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.771145 23342 fs_manager.cc:359] Metadata directory not provided
20:17:59.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.771409 23342 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:17:59.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.779543 23342 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
20:17:59.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.784080 23362 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:17:59.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.785079 23342 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
20:17:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.785458 23342 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data,/tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbb55c2886a415eb3483ad9bc885ba8"
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:17:53 on dist-test-slave-8n49"
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.787863 23342 fs_report.cc:383] FS layout report
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/wal
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/data
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:17:59.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:17:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:17:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:17:59.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.825380 23342 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:17:59.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.832221 23342 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:17:59.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.832805 23342 kserver.cc:163] Server-wide thread pool size limit: 3276
20:17:59.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.905048 23342 rpc_server.cc:307] RPC server started. Bound to: 127.22.90.188:35103
20:17:59.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.905172 23415 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.90.188:35103 every 8 connection(s)
20:17:59.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.908100 23342 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskJJ3Nd5/test-tmp/mini-kudu-cluster3978565703472453433/master-2/data/info.pb
20:17:59.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.909149 22890 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu as pid 23342
20:17:59.910 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:17:59.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.923285 23416 sys_catalog.cc:263] Verifying existing consensus state
20:17:59.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.931396 23416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Bootstrap starting.
20:17:59.976 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:17:59.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.976547 23416 log.cc:824] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Log is configured to *not* fsync() on all Append() calls
20:17:59.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.987571 23416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:17:59.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:17:59.988541 23416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Bootstrap complete.
20:18:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.024219 23416 raft_consensus.cc:348] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:00.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.027704 23416 raft_consensus.cc:374] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:18:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.028031 23416 raft_consensus.cc:729] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfbb55c2886a415eb3483ad9bc885ba8, State: Initialized, Role: FOLLOWER
20:18:00.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.029101 23416 consensus_queue.cc:260] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:00.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.032533 23420 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } } }
20:18:00.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.033567 23420 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [sys.catalog]: This master's current role is: FOLLOWER
20:18:00.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.034390 23416 sys_catalog.cc:564] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [sys.catalog]: configured and running, proceeding with master startup.
20:18:00.058 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.061 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.90.188:35103(127.22.90.188:35103)] connecting to peer
20:18:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:00.077105 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:00.077463 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:00.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:00.089032 23368 tablet.cc:2369] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:18:00.110 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.117 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.121 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.121 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.123 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.90.188:35103(127.22.90.188:35103)] Successfully connected to peer
20:18:00.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.096230 23342 master_runner.cc:186] Error getting master registration for 127.22.90.190:36727: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111),
20:18:00.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.126538 23342 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:18:00.207 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:18:00.219 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.90.188
20:18:00.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.375537 23420 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:18:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.376092 23420 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:00.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.380781 23420 leader_election.cc:288] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d328412ce517409587d18c320e958811 (127.22.90.190:36727), d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071)
20:18:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:00.381804 23364 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111)
20:18:00.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:00.396147 23364 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d328412ce517409587d18c320e958811 (127.22.90.190:36727): Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111)
20:18:00.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:00.397024 23365 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071): Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:18:00.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.397617 23365 leader_election.cc:302] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbb55c2886a415eb3483ad9bc885ba8; no voters: d223c4adcf0f474fb4fa9fc84a1d28f6, d328412ce517409587d18c320e958811
20:18:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:00.398597 23420 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:18:00.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:18:00 dist-test-slave-8n49 krb5kdc[22903](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047474, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.90.188@KRBTEST.COM
20:18:00.504 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9db5f64a, L:/127.0.0.1:44928 - R:/127.22.90.188:35103] using SASL/GSSAPI
20:18:00.522 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.537 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.540 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.540 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.540 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.543 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.546 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.546 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.547 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.549 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.556 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.558 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.559 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.559 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.560 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.563 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.564 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.566 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.575 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.577 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.577 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.583 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.583 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.585 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.596 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.597 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.598 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.598 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.599 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.601 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.601 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.602 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.604 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.636 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.637 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.638 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.639 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.639 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.642 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.642 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.642 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.644 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.717 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.718 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.721 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.721 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.722 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.726 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.726 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.727 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.727 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.816 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.818 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.823 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.823 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.823 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.825 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:00.856 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:00.858 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.858 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:00.859 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:00.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.862 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:00.864 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:01.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:01.081048 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:01.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:01.081328 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:01.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:01.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.258 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:01.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.265 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:01.396 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:01.397 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:01.400 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.401 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.401 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.401 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.403 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:01.576 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:01.578 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:01.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:01.581 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.581 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.582 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:01.587 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:02.084172 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:02.084478 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:02.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:02.545297 23441 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:18:02.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:02.545778 23441 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:02.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:02.547255 23441 leader_election.cc:288] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d328412ce517409587d18c320e958811 (127.22.90.190:36727), d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071)
20:18:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:02.554011 23364 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d328412ce517409587d18c320e958811 (127.22.90.190:36727): Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111)
20:18:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:02.554481 23365 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071): Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:18:02.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:02.554878 23365 leader_election.cc:302] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbb55c2886a415eb3483ad9bc885ba8; no voters: d223c4adcf0f474fb4fa9fc84a1d28f6, d328412ce517409587d18c320e958811
20:18:02.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:02.555558 23441 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:18:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:03.087456 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:03.087725 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:04.076 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:04.077 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:04.079 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:04.079 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:04.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:04.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:04.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:04.080 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:04.083 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:04.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:04.090662 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:04.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:04.090927 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:04.716 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:04.717 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:04.718 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:04.718 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:04.719 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:04.719 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:04.720 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:04.720 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:04.722 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:05.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:05.093786 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:05.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:05.094048 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:05.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:05.129454 23342 master_runner.cc:186] Error getting master registration for 127.22.90.190:36727: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111),
20:18:05.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:05.129876 23342 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:18:05.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:05.326555 23449 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:18:05.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:05.326994 23449 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:05.328804 23449 leader_election.cc:288] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d328412ce517409587d18c320e958811 (127.22.90.190:36727), d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071)
20:18:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:05.333551 23364 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d328412ce517409587d18c320e958811 (127.22.90.190:36727): Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111)
20:18:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:05.334228 23365 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071): Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:18:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:05.334529 23365 leader_election.cc:302] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbb55c2886a415eb3483ad9bc885ba8; no voters: d223c4adcf0f474fb4fa9fc84a1d28f6, d328412ce517409587d18c320e958811
20:18:05.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:05.335041 23449 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:18:05.876 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:05.878 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:05.879 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:05.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:05.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:05.881 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:05.881 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:05.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:05.883 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:06.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:06.097548 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:06.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:06.097961 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:07.101074 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:07.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:07.101328 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:07.736 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:07.739 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:07.739 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:07.739 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:07.739 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:07.742 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:07.742 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:07.742 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:07.744 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:07.860044 22304 debug-util.cc:398] Leaking SignalData structure 0x7b0800072060 after lost signal to thread 22235
20:18:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:07.860646 22304 debug-util.cc:398] Leaking SignalData structure 0x7b080009d020 after lost signal to thread 22307
20:18:08.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:08.104899 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:08.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:08.105175 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:08.876 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:08.878 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:08.878 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:08.879 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:08.879 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:08.881 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:08.881 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:08.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:08.884 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:09.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:09.108928 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:09.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:09.109289 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:09.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:18:09 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047489, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:18:09.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:09.769229 23358 init.cc:295] Successfully reacquired a new kerberos TGT
20:18:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:10.112246 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:10.112557 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:10.134184 23342 master_runner.cc:186] Error getting master registration for 127.22.90.190:36727: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111),
20:18:10.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:10.134660 23342 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:18:10.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:10.369988 23458 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:18:10.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:10.370489 23458 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:10.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:10.372387 23458 leader_election.cc:288] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d328412ce517409587d18c320e958811 (127.22.90.190:36727), d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071)
20:18:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:10.373751 23364 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:18:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:10.376962 23364 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d328412ce517409587d18c320e958811 (127.22.90.190:36727): Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111)
20:18:10.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:10.377478 23365 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071): Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:18:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:10.377763 23365 leader_election.cc:302] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbb55c2886a415eb3483ad9bc885ba8; no voters: d223c4adcf0f474fb4fa9fc84a1d28f6, d328412ce517409587d18c320e958811
20:18:10.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:10.378305 23458 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:18:10.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:10.538 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:10.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:10.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:10.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:10.541 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:10.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:10.541 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:10.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:11.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:11.116135 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:11.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:11.116474 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:12.119863 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:12.120141 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:13.122845 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:13.123138 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:13.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:13.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.260 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:13.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.263 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.263 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.263 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.265 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:13.437 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:13.438 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:13.439 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.439 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.439 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.440 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.440 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.440 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.442 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:13.636 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:13.637 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:13.637 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.638 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.638 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:13.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.640 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:13.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:14.116 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:14.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:14.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:14.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:14.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:14.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:14.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:14.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:14.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:14.125872 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:14.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:14.126164 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:14.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:14.434366 23464 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:18:14.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:14.434778 23464 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:14.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:14.436453 23464 leader_election.cc:288] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d328412ce517409587d18c320e958811 (127.22.90.190:36727), d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071)
20:18:14.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:14.440903 23364 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d328412ce517409587d18c320e958811 (127.22.90.190:36727): Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111)
20:18:14.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:14.441334 23365 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071): Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:18:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:14.441730 23365 leader_election.cc:302] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbb55c2886a415eb3483ad9bc885ba8; no voters: d223c4adcf0f474fb4fa9fc84a1d28f6, d328412ce517409587d18c320e958811
20:18:14.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:14.442320 23464 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:18:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:15.129101 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:15.129410 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:15.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:15.137562 23342 master_runner.cc:186] Error getting master registration for 127.22.90.190:36727: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111),
20:18:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:15.137888 23342 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:18:15.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:15.257 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:15.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:15.257 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:15.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:15.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:15.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:15.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:15.261 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:16.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:16.132622 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:16.132865 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:17.135798 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:17.136080 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:17.216 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:17.219 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:17.219 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:17.219 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:17.220 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:17.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:17.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:17.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:17.225 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:18.138900 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:18.139155 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:19.116 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:19.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:19.118 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:19.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:19.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:19.119 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:19.119 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:19.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:19.122 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:19.142362 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:19.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:19.142683 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:18:19 dist-test-slave-8n49 krb5kdc[22903](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047499, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.90.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:18:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:19.771831 23358 init.cc:295] Successfully reacquired a new kerberos TGT
20:18:20.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:20.141026 23342 master_runner.cc:186] Error getting master registration for 127.22.90.190:36727: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111),
20:18:20.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:20.141422 23342 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:18:20.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:20.145470 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:20.145753 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:21.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:21.148517 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:21.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:21.148824 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:22.037 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:22.038 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:22.038 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:22.038 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:22.038 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:22.039 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:22.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:22.039 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:22.041 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:22.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:22.151690 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:22.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:22.151934 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:23.154675 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:23.154935 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:24.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:24.157660 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:24.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:24.157910 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:24.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:24.930092 23480 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:18:24.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:24.930627 23480 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d328412ce517409587d18c320e958811" member_type: VOTER last_known_addr { host: "127.22.90.190" port: 36727 } } peers { permanent_uuid: "d223c4adcf0f474fb4fa9fc84a1d28f6" member_type: VOTER last_known_addr { host: "127.22.90.189" port: 36071 } } peers { permanent_uuid: "dfbb55c2886a415eb3483ad9bc885ba8" member_type: VOTER last_known_addr { host: "127.22.90.188" port: 35103 } }
20:18:24.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:24.932544 23480 leader_election.cc:288] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d328412ce517409587d18c320e958811 (127.22.90.190:36727), d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071)
20:18:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:24.934280 23364 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:18:24.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:24.937233 23364 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d328412ce517409587d18c320e958811 (127.22.90.190:36727): Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111)
20:18:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:24.937749 23365 leader_election.cc:334] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d223c4adcf0f474fb4fa9fc84a1d28f6 (127.22.90.189:36071): Network error: Client connection negotiation failed: client connection to 127.22.90.189:36071: connect: Connection refused (error 111)
20:18:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:24.938068 23365 leader_election.cc:302] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbb55c2886a415eb3483ad9bc885ba8; no voters: d223c4adcf0f474fb4fa9fc84a1d28f6, d328412ce517409587d18c320e958811
20:18:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:24.938616 23480 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:18:25.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:25.143389 23342 master_runner.cc:186] Error getting master registration for 127.22.90.190:36727: Network error: Client connection negotiation failed: client connection to 127.22.90.190:36727: connect: Connection refused (error 111),
20:18:25.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:25.143726 23342 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:18:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:25.160713 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:25.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:25.160944 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:25.876 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:25.877 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:25.877 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:25.877 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:25.878 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:25.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:25.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:25.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:25.881 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:26.163673 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:26.163924 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:26.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:26.480289 22385 debug-util.cc:398] Leaking SignalData structure 0x7b08000903c0 after lost signal to thread 22317
20:18:26.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:26.481124 22385 debug-util.cc:398] Leaking SignalData structure 0x7b08000c1f80 after lost signal to thread 22388
20:18:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:27.166937 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:27.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:27.167246 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:28.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:28.170234 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:28.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:28.170531 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:28.977 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:28.977 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:28.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:28.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:28.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:28.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:28.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:28.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:28.982 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:29.173846 23433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbb55c2886a415eb3483ad9bc885ba8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:18:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:18:29.174221 23433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:18:29.277 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.189:36071(127.22.90.189:36071)] connecting to peer
20:18:29.278 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.90.190:36727(127.22.90.190:36727)] connecting to peer
20:18:29.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:29.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.90.189:36071(127.22.90.189:36071)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:29.278 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.189:36071: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071
20:18:29.279 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:29.279 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.190:36727(127.22.90.190:36727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:29.279 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.90.190:36727: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
20:18:29.282 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103; will retry
20:18:29.286 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=29368), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [197ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [596ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [625ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [638ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [657ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [662ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [678ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [680ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [714ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [718ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [721ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [800ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [806ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [894ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [900ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [937ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [940ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [943ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [944ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1334ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1337ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1474ms] refreshing cache from master, [1475ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1482ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1654ms] refreshing cache from master, [1655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1658ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [4154ms] refreshing cache from master, [4155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [4158ms] too many traces: truncated at 100 traces, deferred=Deferred@849057494(state=PENDING, result=null, callback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778), errback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778)))
20:18:29.291 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.90.189:36071(127.22.90.189:36071)] disconnecting while in state TERMINATED
20:18:29.292 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.90.190:36727(127.22.90.190:36727)] disconnecting while in state TERMINATED
20:18:29.292 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.90.188:35103(127.22.90.188:35103)] disconnecting while in state READY
20:18:29.298 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:18:29.299 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.90.188:35103(127.22.90.188:35103)] handling channelInactive
20:18:29.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.90.188:35103(127.22.90.188:35103)] cleaning up while in state READY due to: connection closed
20:18:29.300 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:18:29.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:29.308524 22890 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 23342
20:18:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:18:30Z chronyd exiting
20:18:31.537 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:18:31.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:31.538687 22148 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22446
20:18:31.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:31.567083 22148 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22595
20:18:31.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:31.598392 22148 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22745
20:18:31.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:31.634757 22148 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22160
20:18:31.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:31.669183 22148 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22231
20:18:31.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:18:31.701936 22148 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskJJ3Nd5/build/tsan/bin/kudu with pid 22310
20:18:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:18:31Z chronyd exiting
20:18:32.855 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=29368), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [197ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [596ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [625ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [638ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [657ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [662ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [678ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [680ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [714ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [718ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [721ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [800ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [806ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [894ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [900ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [937ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [940ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [943ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [944ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1334ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1337ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1474ms] refreshing cache from master, [1475ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1482ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1654ms] refreshing cache from master, [1655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1658ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [4154ms] refreshing cache from master, [4155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [4158ms] too many traces: truncated at 100 traces, deferred=Deferred@849057494(state=PENDING, result=null, callback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778), errback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.90.190:36727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=29368), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [197ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [596ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [625ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [638ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [657ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [662ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [678ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [680ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [714ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [718ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [721ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [800ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [806ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [894ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [900ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [937ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [940ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [943ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [944ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1334ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1337ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1474ms] refreshing cache from master, [1475ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1482ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1654ms] refreshing cache from master, [1655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1658ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [4154ms] refreshing cache from master, [4155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [4158ms] too many traces: truncated at 100 traces, deferred=Deferred@849057494(state=PENDING, result=null, callback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778), errback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.90.190:36727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
20:18:33.317 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
20:18:33.317 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=29368), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [197ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [596ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [625ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [638ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [657ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [662ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [678ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [680ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [714ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [718ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [721ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [800ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [806ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [894ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [900ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [937ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [940ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [943ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [944ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1334ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1337ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1474ms] refreshing cache from master, [1475ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1482ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1654ms] refreshing cache from master, [1655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1658ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [4154ms] refreshing cache from master, [4155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [4158ms] too many traces: truncated at 100 traces, deferred=Deferred@849057494(state=PENDING, result=null, callback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778), errback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.90.190:36727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=29, TimeoutTracker(timeout=30000, elapsed=29368), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [197ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [191ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [596ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [601ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [619ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [625ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [638ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [645ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [654ms] refreshing cache from master, [654ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [657ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [662ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [674ms] refreshing cache from master, [674ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [678ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [680ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [683ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [714ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [718ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [721ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [723ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [724ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [800ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [806ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [894ms] refreshing cache from master, [895ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [900ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [902ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [904ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [904ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [937ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [940ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [943ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [944ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1334ms] refreshing cache from master, [1335ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1337ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1339ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1345ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1474ms] refreshing cache from master, [1475ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1482ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1654ms] refreshing cache from master, [1655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [1657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [1658ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.189:36071: Network error: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071, [1661ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.190:36727: Network error: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [1661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [1665ms] Sub RPC ConnectToMaster: received response from server master-127.22.90.188:35103: OK, [1666ms] delaying RPC due to: Service unavailable: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727, [4154ms] refreshing cache from master, [4155ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.189:36071, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.190:36727, [4156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.90.188:35103, [4158ms] too many traces: truncated at 100 traces, deferred=Deferred@849057494(state=PENDING, result=null, callback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778), errback=(continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294) -> (continuation of Deferred@1104406497 after retry RPC after error@811117334) -> (continuation of Deferred@1985073775 after retry RPC after error@1309415747) -> (continuation of Deferred@280888204 after retry RPC after error@1699416726) -> (continuation of Deferred@502361850 after retry RPC after error@865708550) -> (continuation of Deferred@1398504858 after retry RPC after error@1853473501) -> (continuation of Deferred@2020206422 after retry RPC after error@115260359) -> (continuation of Deferred@241279275 after retry RPC after error@774825232) -> (continuation of Deferred@424338644 after retry RPC after error@1790534841) -> (continuation of Deferred@1775429692 after retry RPC after error@814585459) -> (continuation of Deferred@2125441266 after retry RPC after error@1595049778)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.90.189:36071,127.22.90.190:36727,127.22.90.188:35103) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.189:36071(127.22.90.189:36071): Connection refused: /127.22.90.189:36071,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.90.190:36727(127.22.90.190:36727): Connection refused: /127.22.90.190:36727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.90.190:36727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)