03:54:09.260 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:54:09.272 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasksR1KPn/test-tmp
03:54:09.275 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:54:09.278 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasksR1KPn/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:54:09.521 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:54:09.664 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:54:09.665 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:54:09.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:09.670969 2774 mini_oidc.cc:152] Starting JWKS server
03:54:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:09.675483 2774 webserver.cc:460] Webserver started at http://127.0.0.1:36231/ using document root <none> and password file <none>
03:54:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:09.676120 2774 mini_oidc.cc:189] Starting OIDC Discovery server
03:54:09.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:09.678264 2774 webserver.cc:460] Webserver started at http://127.0.0.1:46505/ using document root <none> and password file <none>
03:54:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:54:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:54:09.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:54:09Z Disabled control of system clock
03:54:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:09.731531 2774 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/wal
03:54:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data
03:54:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:09.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/logs
03:54:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data/info.pb
03:54:09.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:09.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.190:32969
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.190
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:09.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:09.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36231/jwks.json with env {}
03:54:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.032964 2786 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:10.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.033491 2786 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:10.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.034104 2786 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.072662 2786 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.072940 2786 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.073127 2786 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:10.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.073302 2786 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:10.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.073630 2786 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:10.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.073858 2786 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36231/jwks.json
03:54:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:10.108735 2786 master_runner.cc:386] Master server non-default flags:
03:54:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:10.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data
03:54:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/wal
03:54:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:10.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.190:32969
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36231/jwks.json
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data/info.pb
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.190
03:54:10.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/logs
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:10.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:10.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:10.109948 2786 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:10.111517 2786 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:10.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.128057 2793 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:10.130141 2786 server_base.cc:1034] running on GCE node
03:54:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.128274 2795 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:11.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:10.128062 2792 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:11.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.342682 2786 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.345842 2786 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.347290 2786 hybrid_clock.cc:648] HybridClock initialized: now 1790567651347242 us; error 62 us; skew 500 ppm
03:54:11.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.348117 2786 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:11.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.355096 2786 webserver.cc:460] Webserver started at http://127.2.181.190:40081/ using document root <none> and password file <none>
03:54:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.355962 2786 fs_manager.cc:359] Metadata directory not provided
03:54:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.356148 2786 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:11.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.356590 2786 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.361547 2786 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data/instance:
03:54:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "162dd0bdee884817953b547524ee7b94"
03:54:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:11 on dist-test-slave-j019"
03:54:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.362681 2786 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/wal/instance:
03:54:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "162dd0bdee884817953b547524ee7b94"
03:54:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:11 on dist-test-slave-j019"
03:54:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.370249 2786 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
03:54:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.376221 2802 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.377514 2786 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:54:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.377823 2786 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/wal
03:54:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "162dd0bdee884817953b547524ee7b94"
03:54:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:11 on dist-test-slave-j019"
03:54:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.378208 2786 fs_report.cc:383] FS layout report
03:54:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/wal
03:54:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/wal
03:54:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data/data
03:54:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:11.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.417758 2786 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.419309 2786 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:11.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.419756 2786 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:11.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.504004 2786 rpc_server.cc:307] RPC server started. Bound to: 127.2.181.190:32969
03:54:11.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.504102 2853 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.181.190:32969 every 8 connection(s)
03:54:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.506692 2786 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/data/info.pb
03:54:11.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.511847 2854 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:54:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.517628 2774 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 2786
03:54:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.518204 2774 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-0/wal/instance
03:54:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.523825 2774 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/wal
03:54:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data
03:54:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/logs
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data/info.pb
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:11.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.189:45703
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.189
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:11.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36231/jwks.json with env {}
03:54:11.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.528832 2854 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } has no permanent_uuid. Determining permanent_uuid...
03:54:11.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.562649 2854 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } has no permanent_uuid. Determining permanent_uuid...
03:54:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.565060 2804 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:11.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.569360 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.189:45703: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.618903 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } attempt: 1
03:54:11.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.623100 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.189:45703: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.691527 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } attempt: 2
03:54:11.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.695736 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.189:45703: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.787195 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } attempt: 3
03:54:11.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.790877 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.189:45703: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.835204 2856 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.835777 2856 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:11.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.836418 2856 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:11.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.867522 2856 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.867812 2856 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.867998 2856 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.868234 2856 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.868469 2856 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.868726 2856 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36231/jwks.json
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.905668 2856 master_runner.cc:386] Master server non-default flags:
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/wal
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:11.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.189:45703
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36231/jwks.json
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:11.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data/info.pb
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.189
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/logs
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:11.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.906869 2856 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.908679 2856 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:11.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.931504 2868 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:11.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:11.934309 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } attempt: 4
03:54:11.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.937911 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.189:45703: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:12.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:12.237325 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } attempt: 5
03:54:12.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:12.241622 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.189:45703: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:12.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:12.789400 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } attempt: 6
03:54:12.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:12.799389 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.189:45703: Network error: Client connection negotiation failed: client connection to 127.2.181.189:45703: connect: Connection refused (error 111)
03:54:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.931536 2865 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.122974 2867 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:54:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:11.931515 2866 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.123589 2856 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:54:13.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.127610 2856 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:13.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.131119 2856 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:13.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.132617 2856 hybrid_clock.cc:648] HybridClock initialized: now 1790567653132524 us; error 105 us; skew 500 ppm
03:54:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.133504 2856 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:13.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.139870 2856 webserver.cc:460] Webserver started at http://127.2.181.189:34861/ using document root <none> and password file <none>
03:54:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.140929 2856 fs_manager.cc:359] Metadata directory not provided
03:54:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.141168 2856 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:13.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.141734 2856 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:13.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.146456 2856 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data/instance:
03:54:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12007c2fa6b24a8eacf7da876a0e60ba"
03:54:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:13 on dist-test-slave-j019"
03:54:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.147756 2856 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/wal/instance:
03:54:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12007c2fa6b24a8eacf7da876a0e60ba"
03:54:13.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:13 on dist-test-slave-j019"
03:54:13.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.154754 2856 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
03:54:13.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.160442 2877 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:13.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.161525 2856 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:54:13.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.161870 2856 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/wal
03:54:13.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12007c2fa6b24a8eacf7da876a0e60ba"
03:54:13.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:13 on dist-test-slave-j019"
03:54:13.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.162230 2856 fs_report.cc:383] FS layout report
03:54:13.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/wal
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/wal
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data/data
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:13.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.208197 2856 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.209894 2856 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.210505 2856 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.280112 2856 rpc_server.cc:307] RPC server started. Bound to: 127.2.181.189:45703
03:54:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.280241 2928 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.181.189:45703 every 8 connection(s)
03:54:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.283071 2856 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/data/info.pb
03:54:13.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.288118 2929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:54:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.290625 2774 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 2856
03:54:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.290971 2774 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-1/wal/instance
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.292992 2774 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/wal
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/logs
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data/info.pb
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:13.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.188:32981
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.188
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36231/jwks.json with env {}
03:54:13.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.301069 2929 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } has no permanent_uuid. Determining permanent_uuid...
03:54:13.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.323359 2929 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } has no permanent_uuid. Determining permanent_uuid...
03:54:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.340140 2929 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } has no permanent_uuid. Determining permanent_uuid...
03:54:13.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.343026 2878 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.346817 2929 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.396306 2929 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 1
03:54:13.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.400940 2929 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:13.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.469419 2929 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 2
03:54:13.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.473203 2929 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:13.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.564670 2929 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 3
03:54:13.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.568290 2929 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.608541 2931 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.609093 2931 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.609563 2931 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:13.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.639570 2931 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.639840 2931 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.640039 2931 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.640211 2931 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.640381 2931 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:13.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.640542 2931 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36231/jwks.json
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.674810 2931 master_runner.cc:386] Master server non-default flags:
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/wal
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:13.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.188:32981
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36231/jwks.json
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data/info.pb
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.188
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:13.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/logs
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:13.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:13.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:13.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.676054 2931 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:13.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.678162 2931 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:13.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.697062 2941 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:13.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.711764 2929 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 4
03:54:13.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.716463 2929 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:13.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.860101 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } attempt: 7
03:54:13.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.897025 2854 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } has no permanent_uuid. Determining permanent_uuid...
03:54:13.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.905251 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:13.963959 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 1
03:54:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.969802 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:14.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.015961 2929 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 5
03:54:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.021868 2929 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:14.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.051429 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 2
03:54:14.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.057322 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:14.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.142930 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 3
03:54:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.149310 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.289904 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 4
03:54:14.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.297585 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:14.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.569414 2929 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 6
03:54:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.579973 2929 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.581187 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 5
03:54:14.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.587098 2854 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.2.181.188:32981: Network error: Client connection negotiation failed: client connection to 127.2.181.188:32981: connect: Connection refused (error 111)
03:54:14.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:13.697090 2942 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:14.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.955142 2944 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:14.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:14.957394 2943 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1258 milliseconds
03:54:14.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.957525 2931 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:54:14.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.958878 2931 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.961953 2931 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.963385 2931 hybrid_clock.cc:648] HybridClock initialized: now 1790567654963343 us; error 51 us; skew 500 ppm
03:54:14.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.964128 2931 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.974447 2931 webserver.cc:460] Webserver started at http://127.2.181.188:39921/ using document root <none> and password file <none>
03:54:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.975389 2931 fs_manager.cc:359] Metadata directory not provided
03:54:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.975566 2931 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:14.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.975983 2931 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.980708 2931 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data/instance:
03:54:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa628cadbfc143639a54297b85682422"
03:54:14.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:14 on dist-test-slave-j019"
03:54:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.981758 2931 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/wal/instance:
03:54:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa628cadbfc143639a54297b85682422"
03:54:14.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:14 on dist-test-slave-j019"
03:54:14.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.988618 2931 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
03:54:14.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.993829 2956 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.994788 2931 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
03:54:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.995139 2931 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/wal
03:54:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa628cadbfc143639a54297b85682422"
03:54:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:14 on dist-test-slave-j019"
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:14.995529 2931 fs_report.cc:383] FS layout report
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/wal
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/wal
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data/data
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:14.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:14.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.036831 2931 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.038404 2931 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:15.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.038928 2931 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.108232 2931 rpc_server.cc:307] RPC server started. Bound to: 127.2.181.188:32981
03:54:15.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.108388 3007 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.181.188:32981 every 8 connection(s)
03:54:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.110898 2931 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/data/info.pb
03:54:15.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.116142 3008 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:54:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.117805 2774 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 2931
03:54:15.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.118213 2774 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/master-2/wal/instance
03:54:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.130393 3008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } has no permanent_uuid. Determining permanent_uuid...
03:54:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.139633 2854 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 6
03:54:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.165953 3008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } has no permanent_uuid. Determining permanent_uuid...
03:54:15.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.180265 3008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } has no permanent_uuid. Determining permanent_uuid...
03:54:15.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.183887 2854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: Bootstrap starting.
03:54:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.191819 2854 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: Neither blocks nor log segments found. Creating new log.
03:54:15.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.194020 2854 log.cc:824] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: Log is configured to *not* fsync() on all Append() calls
03:54:15.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.199031 2854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: No bootstrap required, opened a new log
03:54:15.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.200771 3008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422: Bootstrap starting.
03:54:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.206444 3008 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422: Neither blocks nor log segments found. Creating new log.
03:54:15.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.208847 3008 log.cc:824] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422: Log is configured to *not* fsync() on all Append() calls
03:54:15.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.213246 3008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422: No bootstrap required, opened a new log
03:54:15.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.222102 2854 raft_consensus.cc:348] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.222682 2854 raft_consensus.cc:374] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:54:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.222855 2854 raft_consensus.cc:729] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 162dd0bdee884817953b547524ee7b94, State: Initialized, Role: FOLLOWER
03:54:15.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.223698 2854 consensus_queue.cc:260] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.226946 3020 sys_catalog.cc:455] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.228065 3020 sys_catalog.cc:458] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [sys.catalog]: This master's current role is: FOLLOWER
03:54:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.228856 2854 sys_catalog.cc:564] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [sys.catalog]: configured and running, proceeding with master startup.
03:54:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.236346 3008 raft_consensus.cc:348] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.237092 3008 raft_consensus.cc:374] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:54:15.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.237381 3008 raft_consensus.cc:729] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: aa628cadbfc143639a54297b85682422, State: Initialized, Role: FOLLOWER
03:54:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.238605 3008 consensus_queue.cc:260] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.243366 3026 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.244952 3008 sys_catalog.cc:564] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: configured and running, proceeding with master startup.
03:54:15.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.245239 3026 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: This master's current role is: FOLLOWER
03:54:15.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:15.262507 3035 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:15.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:15.263192 3035 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:15.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:15.294770 2962 tablet.cc:2369] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:54:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:15.319185 3044 catalog_manager.cc:1558] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:15.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:15.319842 3044 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:15.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.261621 2786 master_runner.cc:186] Error getting master registration for 127.2.181.189:45703: OK, instance_id { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" instance_seqno: 1790567653260771 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:54:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.322191 2786 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:15.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.288225 2931 master_runner.cc:186] Error getting master registration for 127.2.181.189:45703: OK, instance_id { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" instance_seqno: 1790567653260771 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:54:15.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.335080 2931 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:15.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.349428 3026 raft_consensus.cc:482] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.349947 3026 raft_consensus.cc:504] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.352456 3026 leader_election.cc:288] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 162dd0bdee884817953b547524ee7b94 (127.2.181.190:32969), 12007c2fa6b24a8eacf7da876a0e60ba (127.2.181.189:45703)
03:54:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.353879 2829 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa628cadbfc143639a54297b85682422" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "162dd0bdee884817953b547524ee7b94" is_pre_election: true
03:54:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.354727 2829 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate aa628cadbfc143639a54297b85682422 in term 0.
03:54:15.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.356092 2958 leader_election.cc:302] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 162dd0bdee884817953b547524ee7b94, aa628cadbfc143639a54297b85682422; no voters:
03:54:15.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.357484 3026 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:54:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.357800 3026 raft_consensus.cc:482] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:54:15.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.358809 3026 raft_consensus.cc:3037] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 0 FOLLOWER]: Advancing to term 1
03:54:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.365569 3026 raft_consensus.cc:504] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.376336 3026 leader_election.cc:288] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [CANDIDATE]: Term 1 election: Requested vote from peers 162dd0bdee884817953b547524ee7b94 (127.2.181.190:32969), 12007c2fa6b24a8eacf7da876a0e60ba (127.2.181.189:45703)
03:54:15.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.376377 2829 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa628cadbfc143639a54297b85682422" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "162dd0bdee884817953b547524ee7b94"
03:54:15.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.377020 2829 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [term 0 FOLLOWER]: Advancing to term 1
03:54:15.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.383803 2829 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate aa628cadbfc143639a54297b85682422 in term 1.
03:54:15.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.384891 2958 leader_election.cc:302] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 162dd0bdee884817953b547524ee7b94, aa628cadbfc143639a54297b85682422; no voters:
03:54:15.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.385818 3026 raft_consensus.cc:2781] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 1 FOLLOWER]: Leader election won for term 1
03:54:15.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.396993 3026 raft_consensus.cc:686] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [term 1 LEADER]: Becoming Leader. State: Replica: aa628cadbfc143639a54297b85682422, State: Running, Role: LEADER
03:54:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.398077 3026 consensus_queue.cc:237] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.413939 3046 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa628cadbfc143639a54297b85682422. Latest consensus state: current_term: 1 leader_uuid: "aa628cadbfc143639a54297b85682422" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.414794 3046 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: This master's current role is: LEADER
03:54:15.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.418248 3048 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:54:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.425561 3048 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:54:15.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:15.438573 2808 tablet.cc:2369] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:54:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.457787 2829 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [term 1 FOLLOWER]: Refusing update from remote peer aa628cadbfc143639a54297b85682422: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:54:15.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.459617 3046 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [LEADER]: Connected to new peer: Peer: permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:54:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.501933 3020 sys_catalog.cc:455] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa628cadbfc143639a54297b85682422. Latest consensus state: current_term: 1 leader_uuid: "aa628cadbfc143639a54297b85682422" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.503515 3020 sys_catalog.cc:458] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [sys.catalog]: This master's current role is: FOLLOWER
03:54:15.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.515426 3026 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aa628cadbfc143639a54297b85682422" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.516258 3026 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: This master's current role is: LEADER
03:54:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.533268 3048 catalog_manager.cc:1347] Generated new cluster ID: be27dee5c58348a5934dd463ef8e6784
03:54:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.533578 3048 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:54:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.574153 3020 sys_catalog.cc:455] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aa628cadbfc143639a54297b85682422" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.574957 3020 sys_catalog.cc:458] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94 [sys.catalog]: This master's current role is: FOLLOWER
03:54:15.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.632241 3048 catalog_manager.cc:1370] Generated new certificate authority record
03:54:15.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.634902 3048 catalog_manager.cc:1504] Loading token signing keys...
03:54:15.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.640480 2929 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } attempt: 7
03:54:15.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.671434 3048 catalog_manager.cc:5868] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422: Generated new TSK 0
03:54:15.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.672600 3048 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:54:15.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.706704 2929 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba: Bootstrap starting.
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.711342 2774 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/wal
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/logs
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data/info.pb
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:15.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.129:0
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.181.129
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.129
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:15.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:54:15.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.713748 2929 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba: Neither blocks nor log segments found. Creating new log.
03:54:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.716769 2929 log.cc:824] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba: Log is configured to *not* fsync() on all Append() calls
03:54:15.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.742710 2929 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba: No bootstrap required, opened a new log
03:54:15.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.824287 2929 raft_consensus.cc:348] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.826719 2929 raft_consensus.cc:374] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:54:15.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.827747 2929 raft_consensus.cc:729] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12007c2fa6b24a8eacf7da876a0e60ba, State: Initialized, Role: FOLLOWER
03:54:15.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.830586 2929 consensus_queue.cc:260] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } }
03:54:15.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.841536 3058 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.842368 3058 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [sys.catalog]: This master's current role is: FOLLOWER
03:54:15.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.848886 2902 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [term 0 FOLLOWER]: Advancing to term 1
03:54:15.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.852744 2903 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa628cadbfc143639a54297b85682422" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "12007c2fa6b24a8eacf7da876a0e60ba"
03:54:15.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.854198 2904 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "aa628cadbfc143639a54297b85682422" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" is_pre_election: true
03:54:15.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.860689 2902 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [term 1 FOLLOWER]: Refusing update from remote peer aa628cadbfc143639a54297b85682422: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:54:15.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.862466 3046 consensus_queue.cc:1035] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [LEADER]: Connected to new peer: Peer: permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:54:15.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.872929 2929 sys_catalog.cc:564] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [sys.catalog]: configured and running, proceeding with master startup.
03:54:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.934139 3058 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [sys.catalog]: SysCatalogTable state changed. Reason: New leader aa628cadbfc143639a54297b85682422. Latest consensus state: current_term: 1 leader_uuid: "aa628cadbfc143639a54297b85682422" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.934938 3058 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [sys.catalog]: This master's current role is: FOLLOWER
03:54:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.934686 3064 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "aa628cadbfc143639a54297b85682422" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.936081 3064 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba [sys.catalog]: This master's current role is: FOLLOWER
03:54:15.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:15.984855 3075 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:54:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:16.222675 3057 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:16.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:16.223328 3057 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:16.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:16.224112 3057 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:16.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.269477 3035 catalog_manager.cc:1259] Loaded cluster ID: be27dee5c58348a5934dd463ef8e6784
03:54:16.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.270068 3035 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: loading cluster ID for follower catalog manager: success
03:54:16.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.278285 3035 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: acquiring CA information for follower catalog manager: success
03:54:16.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.285081 3035 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 162dd0bdee884817953b547524ee7b94: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:54:16.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:16.288123 3057 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:16.289858 3057 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.181.129
03:54:16.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.354318 3057 tablet_server_runner.cc:78] Tablet server non-default flags:
03:54:16.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:16.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:16.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:16.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:16.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data
03:54:16.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/wal
03:54:16.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.129:0
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data/info.pb
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.129
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.181.129
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/logs
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:16.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:16.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.355845 3057 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.357800 3057 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:16.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:16.381719 3081 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:16.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.438074 3084 sys_catalog.cc:455] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "aa628cadbfc143639a54297b85682422" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "162dd0bdee884817953b547524ee7b94" member_type: VOTER last_known_addr { host: "127.2.181.190" port: 32969 } } peers { permanent_uuid: "12007c2fa6b24a8eacf7da876a0e60ba" member_type: VOTER last_known_addr { host: "127.2.181.189" port: 45703 } } peers { permanent_uuid: "aa628cadbfc143639a54297b85682422" member_type: VOTER last_known_addr { host: "127.2.181.188" port: 32981 } } }
03:54:16.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:16.439247 3084 sys_catalog.cc:458] T 00000000000000000000000000000000 P aa628cadbfc143639a54297b85682422 [sys.catalog]: This master's current role is: LEADER
03:54:17.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:17.016844 3075 catalog_manager.cc:1259] Loaded cluster ID: be27dee5c58348a5934dd463ef8e6784
03:54:17.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:17.017438 3075 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba: loading cluster ID for follower catalog manager: success
03:54:17.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:17.038362 3075 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba: acquiring CA information for follower catalog manager: success
03:54:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:17.055855 3075 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 12007c2fa6b24a8eacf7da876a0e60ba: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:54:18.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:18.148276 3057 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.769s user 0.493s sys 1.008s
03:54:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:18.155467 3057 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.776s user 0.493s sys 1.009s
03:54:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:17.774832 3080 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 3057
03:54:18.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.162451 3057 server_base.cc:1034] running on GCE node
03:54:18.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:18.166927 3090 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:18.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:16.383585 3082 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:18.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.168696 3057 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:18.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.176981 3057 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.179103 3057 hybrid_clock.cc:648] HybridClock initialized: now 1790567658179070 us; error 59 us; skew 500 ppm
03:54:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.180197 3057 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:18.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.199119 3057 webserver.cc:460] Webserver started at http://127.2.181.129:42707/ using document root <none> and password file <none>
03:54:18.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.200357 3057 fs_manager.cc:359] Metadata directory not provided
03:54:18.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.200672 3057 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:18.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.201205 3057 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.208048 3057 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data/instance:
03:54:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ba27cb228c44f398a55f50e05839717"
03:54:18.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:18 on dist-test-slave-j019"
03:54:18.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.209575 3057 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/wal/instance:
03:54:18.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ba27cb228c44f398a55f50e05839717"
03:54:18.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:18 on dist-test-slave-j019"
03:54:18.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.224733 3057 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.007s sys 0.008s
03:54:18.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.237908 3095 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:18.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.239246 3057 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.000s sys 0.008s
03:54:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.239647 3057 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/wal
03:54:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4ba27cb228c44f398a55f50e05839717"
03:54:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:18 on dist-test-slave-j019"
03:54:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.240088 3057 fs_report.cc:383] FS layout report
03:54:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/wal
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/wal
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data/data
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:18.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:18.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.313592 3057 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:18.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.323323 3057 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:18.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.324186 3057 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.345141 3057 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:54:18.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.359236 3057 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:54:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.359804 3057 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.001s
03:54:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.360250 3057 ts_tablet_manager.cc:610] Registered 0 tablets
03:54:18.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.360592 3057 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:54:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.891451 3057 rpc_server.cc:307] RPC server started. Bound to: 127.2.181.129:40053
03:54:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.891578 3207 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.181.129:40053 every 8 connection(s)
03:54:18.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.894709 3057 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/data/info.pb
03:54:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.903419 2774 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3057
03:54:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.903851 2774 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-0/wal/instance
03:54:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.925361 2774 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/wal
03:54:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data
03:54:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:18.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/logs
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data/info.pb
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.130:0
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.181.130
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.130
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:18.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:54:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.996973 3210 heartbeater.cc:347] Connected to a master server at 127.2.181.190:32969
03:54:18.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.997437 3210 heartbeater.cc:464] Registering TS with master...
03:54:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.998596 3210 heartbeater.cc:511] Master 127.2.181.190:32969 requested a full tablet report, sending...
03:54:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.998750 3209 heartbeater.cc:347] Connected to a master server at 127.2.181.189:45703
03:54:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.999046 3209 heartbeater.cc:464] Registering TS with master...
03:54:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:18.999780 3209 heartbeater.cc:511] Master 127.2.181.189:45703 requested a full tablet report, sending...
03:54:19.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.001852 2818 ts_manager.cc:194] Registered new tserver with Master: 4ba27cb228c44f398a55f50e05839717 (127.2.181.129:40053)
03:54:19.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.003379 3208 heartbeater.cc:347] Connected to a master server at 127.2.181.188:32981
03:54:19.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.003878 2894 ts_manager.cc:194] Registered new tserver with Master: 4ba27cb228c44f398a55f50e05839717 (127.2.181.129:40053)
03:54:19.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.004105 3208 heartbeater.cc:464] Registering TS with master...
03:54:19.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.004870 3208 heartbeater.cc:511] Master 127.2.181.188:32981 requested a full tablet report, sending...
03:54:19.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.006945 2973 ts_manager.cc:194] Registered new tserver with Master: 4ba27cb228c44f398a55f50e05839717 (127.2.181.129:40053)
03:54:19.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.009011 2973 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.181.129:52683
03:54:19.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.338332 3216 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.338915 3216 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.339571 3216 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.371702 3216 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.372833 3216 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.181.130
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.408586 3216 tablet_server_runner.cc:78] Tablet server non-default flags:
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/wal
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.130:0
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data/info.pb
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.130
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:19.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.181.130
03:54:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/logs
03:54:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:54:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.410108 3216 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:19.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:19.412110 3216 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:19.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.437847 3227 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:20.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.012981 3208 heartbeater.cc:503] Master 127.2.181.188:32981 was elected leader, sending a full tablet report...
03:54:20.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.437841 3225 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:20.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:19.440946 3224 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:20.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:20.783890 3226 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1345 milliseconds
03:54:20.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.783998 3216 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:54:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.785279 3216 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:20.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.787990 3216 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:20.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.789453 3216 hybrid_clock.cc:648] HybridClock initialized: now 1790567660789384 us; error 80 us; skew 500 ppm
03:54:20.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.790225 3216 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:20.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.800156 3216 webserver.cc:460] Webserver started at http://127.2.181.130:43389/ using document root <none> and password file <none>
03:54:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.801136 3216 fs_manager.cc:359] Metadata directory not provided
03:54:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.801327 3216 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.801764 3216 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.806720 3216 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data/instance:
03:54:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b35c566b16ab4a768002ada528a3df56"
03:54:20.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:20 on dist-test-slave-j019"
03:54:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.807729 3216 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/wal/instance:
03:54:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b35c566b16ab4a768002ada528a3df56"
03:54:20.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:20 on dist-test-slave-j019"
03:54:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.814824 3216 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
03:54:20.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.820238 3238 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:20.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.821261 3216 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
03:54:20.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.821615 3216 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/wal
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b35c566b16ab4a768002ada528a3df56"
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:20 on dist-test-slave-j019"
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.821938 3216 fs_report.cc:383] FS layout report
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/wal
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/wal
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data/data
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:20.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:20.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.870568 3216 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.872040 3216 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.872604 3216 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:20.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.875381 3216 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:54:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.879405 3216 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:54:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.879628 3216 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:54:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.879961 3216 ts_tablet_manager.cc:610] Registered 0 tablets
03:54:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:20.880141 3216 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:54:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.037741 3216 rpc_server.cc:307] RPC server started. Bound to: 127.2.181.130:41115
03:54:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.037814 3350 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.181.130:41115 every 8 connection(s)
03:54:21.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.040469 3216 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/data/info.pb
03:54:21.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.050365 2774 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3216
03:54:21.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.051163 2774 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-1/wal/instance
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.070538 2774 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/wal
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/logs
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data/info.pb
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:21.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.131:0
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.181.131
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.131
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:54:21.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.107704 3352 heartbeater.cc:347] Connected to a master server at 127.2.181.189:45703
03:54:21.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.108196 3352 heartbeater.cc:464] Registering TS with master...
03:54:21.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.109409 3352 heartbeater.cc:511] Master 127.2.181.189:45703 requested a full tablet report, sending...
03:54:21.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.112418 2894 ts_manager.cc:194] Registered new tserver with Master: b35c566b16ab4a768002ada528a3df56 (127.2.181.130:41115)
03:54:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.114706 3351 heartbeater.cc:347] Connected to a master server at 127.2.181.188:32981
03:54:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.115082 3351 heartbeater.cc:464] Registering TS with master...
03:54:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.115996 3351 heartbeater.cc:511] Master 127.2.181.188:32981 requested a full tablet report, sending...
03:54:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.118070 2973 ts_manager.cc:194] Registered new tserver with Master: b35c566b16ab4a768002ada528a3df56 (127.2.181.130:41115)
03:54:21.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.119514 2973 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.181.130:60659
03:54:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.124353 3353 heartbeater.cc:347] Connected to a master server at 127.2.181.190:32969
03:54:21.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.124662 3353 heartbeater.cc:464] Registering TS with master...
03:54:21.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.125272 3353 heartbeater.cc:511] Master 127.2.181.190:32969 requested a full tablet report, sending...
03:54:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.127808 2818 ts_manager.cc:194] Registered new tserver with Master: b35c566b16ab4a768002ada528a3df56 (127.2.181.130:41115)
03:54:21.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.417339 3359 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.418308 3359 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.418953 3359 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:21.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.453707 3359 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:21.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.455112 3359 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.181.131
03:54:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.490335 3359 tablet_server_runner.cc:78] Tablet server non-default flags:
03:54:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.181.148:37139
03:54:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/wal
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.181.131:0
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data/info.pb
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.181.131
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.181.190:32969,127.2.181.189:45703,127.2.181.188:32981
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.181.131
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/logs
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:21.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.491887 3359 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:21.493534 3359 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:21.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.514704 3367 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:22.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.124534 3351 heartbeater.cc:503] Master 127.2.181.188:32981 was elected leader, sending a full tablet report...
03:54:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.515975 3370 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:22.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:21.514773 3368 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:22.725040 3369 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:54:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.725206 3359 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:54:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.728847 3359 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:22.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.731657 3359 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:22.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.733096 3359 hybrid_clock.cc:648] HybridClock initialized: now 1790567662733056 us; error 61 us; skew 500 ppm
03:54:22.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.733853 3359 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.741663 3359 webserver.cc:460] Webserver started at http://127.2.181.131:38551/ using document root <none> and password file <none>
03:54:22.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.742745 3359 fs_manager.cc:359] Metadata directory not provided
03:54:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.742949 3359 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:22.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.743372 3359 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:22.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.748131 3359 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data/instance:
03:54:22.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea860909c0af4475a0d5fd1845775294"
03:54:22.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:22 on dist-test-slave-j019"
03:54:22.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.749442 3359 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/wal/instance:
03:54:22.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea860909c0af4475a0d5fd1845775294"
03:54:22.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:22 on dist-test-slave-j019"
03:54:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.756772 3359 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:54:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.762414 3378 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.763461 3359 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
03:54:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.763803 3359 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/wal
03:54:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea860909c0af4475a0d5fd1845775294"
03:54:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:22 on dist-test-slave-j019"
03:54:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.764233 3359 fs_report.cc:383] FS layout report
03:54:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/wal
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/wal
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data/data
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:22.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:22.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.811997 3359 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:22.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.813429 3359 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.813838 3359 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:22.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.816439 3359 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:54:22.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.820415 3359 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:54:22.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.820619 3359 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
03:54:22.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.820829 3359 ts_tablet_manager.cc:610] Registered 0 tablets
03:54:22.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.820958 3359 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:54:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.997169 3359 rpc_server.cc:307] RPC server started. Bound to: 127.2.181.131:37673
03:54:22.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.997311 3490 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.2.181.131:37673 every 8 connection(s)
03:54:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:22.999810 3359 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/data/info.pb
03:54:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.000413 2774 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3359
03:54:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.000859 2774 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster7157691563316784481/ts-2/wal/instance
03:54:23.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.050709 3492 heartbeater.cc:347] Connected to a master server at 127.2.181.189:45703
03:54:23.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.051139 3492 heartbeater.cc:464] Registering TS with master...
03:54:23.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.052932 3492 heartbeater.cc:511] Master 127.2.181.189:45703 requested a full tablet report, sending...
03:54:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.054513 3491 heartbeater.cc:347] Connected to a master server at 127.2.181.188:32981
03:54:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.055187 3491 heartbeater.cc:464] Registering TS with master...
03:54:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.055979 3491 heartbeater.cc:511] Master 127.2.181.188:32981 requested a full tablet report, sending...
03:54:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.056073 2894 ts_manager.cc:194] Registered new tserver with Master: ea860909c0af4475a0d5fd1845775294 (127.2.181.131:37673)
03:54:23.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.058552 2973 ts_manager.cc:194] Registered new tserver with Master: ea860909c0af4475a0d5fd1845775294 (127.2.181.131:37673)
03:54:23.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.060566 2973 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.2.181.131:58093
03:54:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.061417 3493 heartbeater.cc:347] Connected to a master server at 127.2.181.190:32969
03:54:23.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.062212 3493 heartbeater.cc:464] Registering TS with master...
03:54:23.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.063366 3493 heartbeater.cc:511] Master 127.2.181.190:32969 requested a full tablet report, sending...
03:54:23.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.066336 2819 ts_manager.cc:194] Registered new tserver with Master: ea860909c0af4475a0d5fd1845775294 (127.2.181.131:37673)
03:54:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.072618 2774 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:54:23.083 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:54:23.084 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:54:23.095 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.181.190"
port: 32969
}
bound_http_address {
host: "127.2.181.190"
port: 40081
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.181.189"
port: 45703
}
bound_http_address {
host: "127.2.181.189"
port: 34861
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.181.188"
port: 32981
}
bound_http_address {
host: "127.2.181.188"
port: 39921
}
}
}
03:54:23.101 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:54:23.104 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.2.181.129"
port: 40053
}
bound_http_address {
host: "127.2.181.129"
port: 42707
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.2.181.130"
port: 41115
}
bound_http_address {
host: "127.2.181.130"
port: 43389
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.2.181.131"
port: 37673
}
bound_http_address {
host: "127.2.181.131"
port: 38551
}
}
}
03:54:23.104 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:54:23.276 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:54:23.387 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasksR1KPn/test-tmp
03:54:23.388 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:54:23.388 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasksR1KPn/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:54:23.393 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:54:23.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:23.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:54:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.747341 3502 mini_oidc.cc:152] Starting JWKS server
03:54:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.751114 3502 webserver.cc:460] Webserver started at http://127.0.0.1:41597/ using document root <none> and password file <none>
03:54:23.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.751524 3502 mini_oidc.cc:189] Starting OIDC Discovery server
03:54:23.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:23.753441 3502 webserver.cc:460] Webserver started at http://127.0.0.1:35535/ using document root <none> and password file <none>
03:54:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:54:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/principal' for realm 'KRBTEST.COM',
03:54:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:54:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:23 dist-test-slave-j019 krb5kdc[3515](info): setting up network...
03:54:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:54:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:23 dist-test-slave-j019 krb5kdc[3515](info): set up 2 sockets
03:54:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:23 dist-test-slave-j019 krb5kdc[3515](info): commencing operation
03:54:23.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:54:24.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:24.066226 3491 heartbeater.cc:503] Master 127.2.181.188:32981 was elected leader, sending a full tablet report...
03:54:25.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.297086 3502 mini_kdc.cc:121] Time spent starting KDC: real 1.518s user 0.004s sys 0.020s
03:54:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:54:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:54:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:54:25.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:54:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:54:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:54:25.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:54:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:54:25.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:54:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:54:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/test-user.keytab.
03:54:25.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/test-user.keytab.
03:54:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:25 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567665, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:54:25.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:54:25Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:54:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:54:25Z Disabled control of system clock
03:54:25.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.107.190@KRBTEST.COM; defaulting to no policy
03:54:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:54:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.107.190@KRBTEST.COM" created.
03:54:25.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:54:25.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.107.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.190.keytab.
03:54:25.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.107.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.190.keytab.
03:54:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:25.501385 3502 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/logs
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/info.pb
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.190:38431
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.190
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.190.keytab
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.190
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:25.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:54:25.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.643164 3487 debug-util.cc:398] Leaking SignalData structure 0x7b08000af300 after lost signal to thread 3362
03:54:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.644042 3487 debug-util.cc:398] Leaking SignalData structure 0x7b08000bebe0 after lost signal to thread 3490
03:54:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.822059 3528 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.822597 3528 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:25.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.823208 3528 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.855813 3528 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:54:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.856033 3528 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.856287 3528 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.856477 3528 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:25.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.856748 3528 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.856976 3528 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:25.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.857188 3528 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41597/jwks.json
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:25.895830 3528 master_runner.cc:386] Master server non-default flags:
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.190.keytab
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.190
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.190:38431
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/info.pb
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.190
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/logs
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:25.897291 3528 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:25.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:25.898741 3528 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.924170 3535 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.925865 3534 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:25.927111 3528 server_base.cc:1034] running on GCE node
03:54:27.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:25.924329 3537 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:27.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.283813 3528 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:27.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.287113 3528 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.288741 3528 hybrid_clock.cc:648] HybridClock initialized: now 1790567667288673 us; error 76 us; skew 500 ppm
03:54:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.289847 3528 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:27 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567667, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.301884 3528 init.cc:377] Logged in from keytab as kudu/127.3.107.190@KRBTEST.COM (short username kudu)
03:54:27.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.308347 3528 webserver.cc:460] Webserver started at http://127.3.107.190:44875/ using document root <none> and password file <none>
03:54:27.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.309634 3528 fs_manager.cc:359] Metadata directory not provided
03:54:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.309903 3528 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.310479 3528 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:27.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.318373 3528 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/instance:
03:54:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c89b68037d064339841d1029aa535857"
03:54:27.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:27 on dist-test-slave-j019"
03:54:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.319871 3528 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal/instance:
03:54:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c89b68037d064339841d1029aa535857"
03:54:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:27 on dist-test-slave-j019"
03:54:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.330130 3528 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.003s
03:54:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.338115 3546 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.339474 3528 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
03:54:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.339962 3528 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c89b68037d064339841d1029aa535857"
03:54:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:27 on dist-test-slave-j019"
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.340453 3528 fs_report.cc:383] FS layout report
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/data
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.385416 3528 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:27.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.393334 3528 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.393908 3528 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.498955 3528 rpc_server.cc:307] RPC server started. Bound to: 127.3.107.190:38431
03:54:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.498986 3598 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.107.190:38431 every 8 connection(s)
03:54:27.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.503314 3528 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/info.pb
03:54:27.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.509768 3502 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3528
03:54:27.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.510062 3599 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:54:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.510754 3502 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal/instance
03:54:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.528038 3599 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } has no permanent_uuid. Determining permanent_uuid...
03:54:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.107.189@KRBTEST.COM; defaulting to no policy
03:54:27.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:54:27.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.107.189@KRBTEST.COM" created.
03:54:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:27 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567667, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.190@KRBTEST.COM for kudu/127.3.107.190@KRBTEST.COM
03:54:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.582840 3599 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } has no permanent_uuid. Determining permanent_uuid...
03:54:27.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.585599 3551 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:27.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:54:27.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.107.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.189.keytab.
03:54:27.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.107.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.189.keytab.
03:54:27.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.589543 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.189:41469: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.590113 3502 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/logs
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/info.pb
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.189:41469
03:54:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.189
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.189.keytab
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.189
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:54:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.638999 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } attempt: 1
03:54:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.643312 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.189:41469: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:27.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.711779 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } attempt: 2
03:54:27.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.715620 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.189:41469: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.807092 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } attempt: 3
03:54:27.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.810945 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.189:41469: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.907891 3606 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.908447 3606 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:27.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.909096 3606 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.940388 3606 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:54:27.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.940734 3606 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.941052 3606 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.941252 3606 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.941437 3606 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:27.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.941619 3606 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:27.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.941874 3606 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41597/jwks.json
03:54:27.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.954383 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } attempt: 4
03:54:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:27.957890 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.189:41469: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.981149 3606 master_runner.cc:386] Master server non-default flags:
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.189.keytab
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.189
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:27.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.189:41469
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/info.pb
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.189
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/logs
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:27.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.982398 3606 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:27.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:27.983449 3606 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:28.001567 3613 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:28.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:28.257282 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } attempt: 5
03:54:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:28.262006 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.189:41469: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:28.809444 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } attempt: 6
03:54:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:28.817444 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.189:41469: Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:29.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:28.002125 3615 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:29.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:28.002619 3612 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:28.004168 3606 server_base.cc:1034] running on GCE node
03:54:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.261109 3606 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:29.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.263613 3606 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.265017 3606 hybrid_clock.cc:648] HybridClock initialized: now 1790567669264973 us; error 64 us; skew 500 ppm
03:54:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.266026 3606 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:29 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567669, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.272128 3606 init.cc:377] Logged in from keytab as kudu/127.3.107.189@KRBTEST.COM (short username kudu)
03:54:29.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.277129 3606 webserver.cc:460] Webserver started at http://127.3.107.189:42963/ using document root <none> and password file <none>
03:54:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.278080 3606 fs_manager.cc:359] Metadata directory not provided
03:54:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.278267 3606 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.278647 3606 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.285344 3606 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/instance:
03:54:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3c8f0a15614ee398009cc1a1cf6645"
03:54:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:29 on dist-test-slave-j019"
03:54:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.286732 3606 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal/instance:
03:54:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3c8f0a15614ee398009cc1a1cf6645"
03:54:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:29 on dist-test-slave-j019"
03:54:29.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.295493 3606 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
03:54:29.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.302544 3625 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.303731 3606 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:54:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.304091 3606 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3c8f0a15614ee398009cc1a1cf6645"
03:54:29.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:29 on dist-test-slave-j019"
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.304482 3606 fs_report.cc:383] FS layout report
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/data
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:29.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.346829 3606 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.352278 3606 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:29.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.352751 3606 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.426549 3606 rpc_server.cc:307] RPC server started. Bound to: 127.3.107.189:41469
03:54:29.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.426613 3677 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.107.189:41469 every 8 connection(s)
03:54:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.429502 3606 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/info.pb
03:54:29.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.431013 3502 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3606
03:54:29.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.431460 3502 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal/instance
03:54:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.435915 3678 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:54:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.3.107.188@KRBTEST.COM; defaulting to no policy
03:54:29.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:54:29.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.3.107.188@KRBTEST.COM" created.
03:54:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.450519 3678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } has no permanent_uuid. Determining permanent_uuid...
03:54:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:29 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567669, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.189@KRBTEST.COM for kudu/127.3.107.190@KRBTEST.COM
03:54:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:54:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.107.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.188.keytab.
03:54:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.3.107.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.188.keytab.
03:54:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.494722 3678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } has no permanent_uuid. Determining permanent_uuid...
03:54:29.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.497570 3502 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:29.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/logs
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/info.pb
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.188:43667
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.188
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.188.keytab
03:54:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.188
03:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:54:29.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:29 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567669, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.189@KRBTEST.COM for kudu/127.3.107.189@KRBTEST.COM
03:54:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.535041 3678 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } has no permanent_uuid. Determining permanent_uuid...
03:54:29.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.537683 3628 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:29.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.542457 3678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:29.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.591997 3678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 1
03:54:29.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.595736 3678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:29.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.664220 3678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 2
03:54:29.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.668265 3678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:29.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.759699 3678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 3
03:54:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.763379 3678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.835177 3685 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.835695 3685 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:29.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.836251 3685 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.868355 3685 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:54:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.868654 3685 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.869011 3685 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.869225 3685 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.869427 3685 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.869619 3685 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.869758 3685 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41597/jwks.json
03:54:29.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.877895 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } attempt: 7
03:54:29.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:29 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567667, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.190@KRBTEST.COM for kudu/127.3.107.189@KRBTEST.COM
03:54:29.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.903252 3599 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } has no permanent_uuid. Determining permanent_uuid...
03:54:29.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.906803 3678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 4
03:54:29.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.908903 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:29.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.910320 3678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.915504 3685 master_runner.cc:386] Master server non-default flags:
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.188.keytab
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.188
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.188:43667
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/info.pb
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.188
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/logs
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.916801 3685 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.917793 3685 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.936715 3698 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.967350 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 1
03:54:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.972203 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:30.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:30.053789 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 2
03:54:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:30.057498 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:30.142849 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 3
03:54:30.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:30.146428 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:30.210129 3678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 5
03:54:30.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:30.214005 3678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:30.286842 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 4
03:54:30.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:30.290297 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:30.573706 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 5
03:54:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:30.578111 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:30.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:30.792246 3678 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 6
03:54:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:30.799324 3678 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:31.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.130544 3599 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } attempt: 6
03:54:31.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:31.141260 3599 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.107.188:43667: Network error: Client connection negotiation failed: client connection to 127.3.107.188:43667: connect: Connection refused (error 111)
03:54:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.936743 3695 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:29.937880 3685 server_base.cc:1034] running on GCE node
03:54:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:29.936830 3696 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.220479 3685 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:31.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.223061 3685 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.224444 3685 hybrid_clock.cc:648] HybridClock initialized: now 1790567671224411 us; error 50 us; skew 500 ppm
03:54:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.225383 3685 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567671, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:31.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.231732 3685 init.cc:377] Logged in from keytab as kudu/127.3.107.188@KRBTEST.COM (short username kudu)
03:54:31.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.238106 3685 webserver.cc:460] Webserver started at http://127.3.107.188:38309/ using document root <none> and password file <none>
03:54:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.239300 3685 fs_manager.cc:359] Metadata directory not provided
03:54:31.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.239553 3685 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:31.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.240098 3685 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:54:31.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.247073 3685 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/instance:
03:54:31.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a98d682389e467b8533c5e05812bae9"
03:54:31.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:31 on dist-test-slave-j019"
03:54:31.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.248538 3685 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal/instance:
03:54:31.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a98d682389e467b8533c5e05812bae9"
03:54:31.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:31 on dist-test-slave-j019"
03:54:31.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.257694 3685 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
03:54:31.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.265409 3711 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:31.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.266613 3685 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.266909 3685 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a98d682389e467b8533c5e05812bae9"
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:31 on dist-test-slave-j019"
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.267206 3685 fs_report.cc:383] FS layout report
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/data
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:31.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:31.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.321574 3685 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:31.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.327718 3685 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:31.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.328161 3685 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.410812 3685 rpc_server.cc:307] RPC server started. Bound to: 127.3.107.188:43667
03:54:31.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.410908 3763 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.107.188:43667 every 8 connection(s)
03:54:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.413430 3685 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/info.pb
03:54:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.418237 3502 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3685
03:54:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.418609 3502 external_mini_cluster.cc:1402] Reading /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal/instance
03:54:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.418738 3764 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:54:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.432937 3764 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } has no permanent_uuid. Determining permanent_uuid...
03:54:31.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567665, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.107.188@KRBTEST.COM
03:54:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567671, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for kudu/127.3.107.190@KRBTEST.COM
03:54:31.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.475339 3764 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } has no permanent_uuid. Determining permanent_uuid...
03:54:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567671, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for kudu/127.3.107.189@KRBTEST.COM
03:54:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567665, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.107.189@KRBTEST.COM
03:54:31.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:31.499295 3674 debug-util.cc:398] Leaking SignalData structure 0x7b08000735c0 after lost signal to thread 3607
03:54:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.505141 3764 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } has no permanent_uuid. Determining permanent_uuid...
03:54:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567665, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.107.190@KRBTEST.COM
03:54:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567671, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for kudu/127.3.107.188@KRBTEST.COM
03:54:31.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.551493 3502 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
03:54:31.558 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:31.559 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:54:31.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.557761 3764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Bootstrap starting.
03:54:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.566030 3764 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Neither blocks nor log segments found. Creating new log.
03:54:31.571 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:54:31.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:54:31.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.568363 3764 log.cc:824] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Log is configured to *not* fsync() on all Append() calls
03:54:31.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.577445 3764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: No bootstrap required, opened a new log
03:54:31.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.107.190"
port: 38431
}
bound_http_address {
host: "127.3.107.190"
port: 44875
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.107.189"
port: 41469
}
bound_http_address {
host: "127.3.107.189"
port: 42963
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.107.188"
port: 43667
}
bound_http_address {
host: "127.3.107.188"
port: 38309
}
}
}
03:54:31.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:54:31.582 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:31.582 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:54:31.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:54:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567671, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:31.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:54:31.613 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:31.615 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc
03:54:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.613088 3764 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.614409 3764 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:54:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.614710 3764 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a98d682389e467b8533c5e05812bae9, State: Initialized, Role: FOLLOWER
03:54:31.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.615710 3764 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:31.626 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:54:31.627 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc
03:54:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.630954 3775 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } } }
03:54:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.632216 3775 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [sys.catalog]: This master's current role is: FOLLOWER
03:54:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.633220 3764 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [sys.catalog]: configured and running, proceeding with master startup.
03:54:31.633 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:54:31.637 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:54:31.639 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:54:31.640 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:54:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:31 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567671, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:54:31.683 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:31.683 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc
03:54:31.685 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:54:31.685 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.107.188:43667
03:54:31.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:54:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.688361 3502 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 3685
03:54:31.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:31.716 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.107.190:38431
03:54:31.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:54:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.716987 3502 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 3528
03:54:31.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:31.741 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.3.107.189:41469
03:54:31.742 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:54:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.742561 3502 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 3606
03:54:31.763 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:31.763 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.107.188:43667
03:54:31.764 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:31.766760 3502 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:31.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/logs
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/info.pb
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.188:43667
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.188
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38309
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.188.keytab
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.188
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:31.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:31.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:31.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:31.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:31.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:31.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:54:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.079641 3785 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:32.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.080447 3785 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:32.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.081502 3785 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.116323 3785 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:54:32.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.116609 3785 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.116900 3785 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.117102 3785 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.117296 3785 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.117482 3785 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:32.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.117619 3785 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41597/jwks.json
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:32.154912 3785 master_runner.cc:386] Master server non-default flags:
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:32.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.188.keytab
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.188
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.188:43667
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:32.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/info.pb
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.188
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38309
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/logs
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:32.156193 3785 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:32.157150 3785 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:32.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.175885 3792 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:33.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.175951 3793 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:33.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:32.177496 3785 server_base.cc:1034] running on GCE node
03:54:33.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:32.176931 3795 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:33.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.434124 3785 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:33.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.437245 3785 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.438680 3785 hybrid_clock.cc:648] HybridClock initialized: now 1790567673438606 us; error 86 us; skew 500 ppm
03:54:33.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.439502 3785 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:33.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:33 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567673, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:33.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.450289 3785 init.cc:377] Logged in from keytab as kudu/127.3.107.188@KRBTEST.COM (short username kudu)
03:54:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.456758 3785 webserver.cc:460] Webserver started at http://127.3.107.188:38309/ using document root <none> and password file <none>
03:54:33.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.457729 3785 fs_manager.cc:359] Metadata directory not provided
03:54:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.457914 3785 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.466114 3785 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
03:54:33.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.470461 3803 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.471648 3785 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
03:54:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.471912 3785 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a98d682389e467b8533c5e05812bae9"
03:54:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:31 on dist-test-slave-j019"
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.474526 3785 fs_report.cc:383] FS layout report
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/wal
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/data
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:33.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.531656 3785 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:33.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.537521 3785 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:33.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.537914 3785 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:33.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.607915 3785 rpc_server.cc:307] RPC server started. Bound to: 127.3.107.188:43667
03:54:33.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.608048 3854 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.107.188:43667 every 8 connection(s)
03:54:33.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.610550 3785 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-2/data/info.pb
03:54:33.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.618391 3502 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3785
03:54:33.619 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:33.619 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.107.190:38431
03:54:33.619 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.620819 3502 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/logs
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/info.pb
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:33.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.190:38431
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.190
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44875
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.190.keytab
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.190
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:33.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:54:33.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.623910 3855 sys_catalog.cc:263] Verifying existing consensus state
03:54:33.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.630422 3855 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Bootstrap starting.
03:54:33.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.663398 3855 log.cc:824] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Log is configured to *not* fsync() on all Append() calls
03:54:33.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.668212 3855 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:54:33.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.669005 3855 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Bootstrap complete.
03:54:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.695551 3855 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.698145 3855 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:54:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.698494 3855 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4a98d682389e467b8533c5e05812bae9, State: Initialized, Role: FOLLOWER
03:54:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.699359 3855 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:33.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.703692 3861 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } } }
03:54:33.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.704663 3861 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [sys.catalog]: This master's current role is: FOLLOWER
03:54:33.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.705480 3855 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [sys.catalog]: configured and running, proceeding with master startup.
03:54:33.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:33.737908 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:33.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:33.738281 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.738368 3785 master_runner.cc:186] Error getting master registration for 127.3.107.190:38431: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111),
03:54:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.770435 3785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:33.793257 3809 tablet.cc:2369] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:54:33.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.817286 3861 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:33.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.817905 3861 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.821375 3861 leader_election.cc:288] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c89b68037d064339841d1029aa535857 (127.3.107.190:38431), 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469)
03:54:33.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:33.828821 3807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:33.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:33.835461 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c89b68037d064339841d1029aa535857 (127.3.107.190:38431): Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:33.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:33.836601 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469): Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.836928 3807 leader_election.cc:302] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a98d682389e467b8533c5e05812bae9; no voters: 6f3c8f0a15614ee398009cc1a1cf6645, c89b68037d064339841d1029aa535857
03:54:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:33.837728 3861 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:54:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.001171 3858 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.001736 3858 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:34.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.002417 3858 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.035177 3858 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:54:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.035532 3858 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.036008 3858 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.036315 3858 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.036705 3858 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.037103 3858 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.037348 3858 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41597/jwks.json
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:34.075892 3858 master_runner.cc:386] Master server non-default flags:
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.190.keytab
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.190
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.190:38431
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/info.pb
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.190
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44875
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/logs
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:34.077421 3858 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:34.078672 3858 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:34.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.102299 3882 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.744690 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.745107 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:35.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:35.580056 3888 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:35.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:35.582296 3888 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:35.601403 3888 leader_election.cc:288] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c89b68037d064339841d1029aa535857 (127.3.107.190:38431), 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469)
03:54:35.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:35.646507 3807 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.3.107.190:38431, user_credentials={real_user=slave}} blocked reactor thread for 48132.5us
03:54:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:35.652849 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c89b68037d064339841d1029aa535857 (127.3.107.190:38431): Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:35.653539 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469): Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:35.654022 3807 leader_election.cc:302] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a98d682389e467b8533c5e05812bae9; no voters: 6f3c8f0a15614ee398009cc1a1cf6645, c89b68037d064339841d1029aa535857
03:54:35.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:35.664770 3888 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:54:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:35.750377 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:35.751145 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:36.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:35.495358 3881 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 3858
03:54:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.171834 3881 kernel_stack_watchdog.cc:198] Thread 3858 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 391ms:
03:54:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:54:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:54:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:54:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:54:36.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:34.102604 3883 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.172904 3858 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.072s user 0.621s sys 1.084s
03:54:36.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.173305 3858 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.073s user 0.621s sys 1.084s
03:54:36.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.178138 3884 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:54:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.178999 3885 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.179083 3858 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:54:36.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.180635 3858 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.183763 3858 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:36.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.185217 3858 hybrid_clock.cc:648] HybridClock initialized: now 1790567676185143 us; error 68 us; skew 500 ppm
03:54:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.186249 3858 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:36 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567676, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:36.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.194276 3858 init.cc:377] Logged in from keytab as kudu/127.3.107.190@KRBTEST.COM (short username kudu)
03:54:36.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.200688 3858 webserver.cc:460] Webserver started at http://127.3.107.190:44875/ using document root <none> and password file <none>
03:54:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.201870 3858 fs_manager.cc:359] Metadata directory not provided
03:54:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.202157 3858 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.212770 3858 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
03:54:36.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.218139 3897 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:36.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.219329 3858 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:54:36.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.219679 3858 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c89b68037d064339841d1029aa535857"
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:27 on dist-test-slave-j019"
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.222939 3858 fs_report.cc:383] FS layout report
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/data
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:36.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:36.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.267040 3858 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.274799 3858 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:36.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.275372 3858 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.369820 3858 rpc_server.cc:307] RPC server started. Bound to: 127.3.107.190:38431
03:54:36.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.369940 3948 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.107.190:38431 every 8 connection(s)
03:54:36.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.373908 3858 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/data/info.pb
03:54:36.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.375285 3502 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3858
03:54:36.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:36.377 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.3.107.189:41469
03:54:36.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.380882 3502 external_mini_cluster.cc:1326] Running /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/logs
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/info.pb
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:54:36.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:36.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.189:41469
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.189
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42963
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.189.keytab
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.189
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:36.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:36.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:54:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.389099 3949 sys_catalog.cc:263] Verifying existing consensus state
03:54:36.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 03:54:36.390141 3949 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:54:36.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.391232 3858 master.cc:537] Master@127.3.107.190:38431 shutting down...
03:54:36.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.469594 3858 master.cc:552] Master@127.3.107.190:38431 shutdown complete.
03:54:36.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:54:36.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.755766 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:36.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.756099 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:36.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.758289 3951 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:54:36.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.758942 3951 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:54:36.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.759673 3951 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:54:36.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.792905 3951 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:54:36.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.793296 3951 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:54:36.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.793733 3951 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:54:36.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.794056 3951 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:54:36.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.794385 3951 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:54:36.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.794699 3951 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:54:36.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.794938 3951 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41597/jwks.json
03:54:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.833442 3951 master_runner.cc:386] Master server non-default flags:
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.107.148:43181
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.107.190:38431,127.3.107.189:41469,127.3.107.188:43667
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/krb5kdc/kudu_127.3.107.189.keytab
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.3.107.189
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.107.189:41469
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:54:36.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41597/jwks.json
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/info.pb
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.107.189
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42963
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:54:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/logs
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 02:50:13 UTC on ccycloud.quasar-purgqw.root.comops.site
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4676527
03:54:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:54:36.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.834918 3951 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:54:36.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:36.836478 3951 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:54:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.860034 3958 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:37.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:37.541712 3963 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:37.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:37.542791 3963 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:37.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:37.551944 3963 leader_election.cc:288] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c89b68037d064339841d1029aa535857 (127.3.107.190:38431), 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469)
03:54:37.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:37.585289 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c89b68037d064339841d1029aa535857 (127.3.107.190:38431): Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:37.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:37.589551 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469): Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:37.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:37.590653 3807 leader_election.cc:302] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a98d682389e467b8533c5e05812bae9; no voters: 6f3c8f0a15614ee398009cc1a1cf6645, c89b68037d064339841d1029aa535857
03:54:37.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:37.592083 3963 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:54:37.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:37.762339 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:37.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:37.763319 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.254668 3956 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 3951
03:54:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.415156 3956 kernel_stack_watchdog.cc:198] Thread 3951 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 393ms:
03:54:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:54:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:54:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:54:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:54:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:36.860270 3957 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:38.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.415990 3951 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.557s user 0.516s sys 0.963s
03:54:38.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.416347 3951 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.558s user 0.516s sys 0.963s
03:54:38.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.418979 3960 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:54:38.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.422473 3959 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1556 milliseconds
03:54:38.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.422618 3951 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:54:38.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.423822 3951 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:54:38.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.426285 3951 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:54:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.427680 3951 hybrid_clock.cc:648] HybridClock initialized: now 1790567678427634 us; error 52 us; skew 500 ppm
03:54:38.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.428794 3951 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:54:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:38 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567678, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:38.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.435729 3951 init.cc:377] Logged in from keytab as kudu/127.3.107.189@KRBTEST.COM (short username kudu)
03:54:38.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.441057 3951 webserver.cc:460] Webserver started at http://127.3.107.189:42963/ using document root <none> and password file <none>
03:54:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.441952 3951 fs_manager.cc:359] Metadata directory not provided
03:54:38.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.442219 3951 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:54:38.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.450197 3951 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
03:54:38.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.454730 3972 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:54:38.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.455760 3951 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:54:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.456102 3951 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data,/tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6f3c8f0a15614ee398009cc1a1cf6645"
03:54:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 03:54:29 on dist-test-slave-j019"
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.458463 3951 fs_report.cc:383] FS layout report
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/data
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:54:38.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:54:38.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.507906 3951 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:54:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.513829 3951 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:54:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.514358 3951 kserver.cc:163] Server-wide thread pool size limit: 3276
03:54:38.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.588039 3951 rpc_server.cc:307] RPC server started. Bound to: 127.3.107.189:41469
03:54:38.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.588135 4023 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.107.189:41469 every 8 connection(s)
03:54:38.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.590823 3951 server_base.cc:1166] Dumped server information to /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/data/info.pb
03:54:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.593837 3502 external_mini_cluster.cc:1388] Started /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu as pid 3951
03:54:38.594 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:54:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.602303 4024 sys_catalog.cc:263] Verifying existing consensus state
03:54:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 03:54:38.602946 4024 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:54:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.603312 3951 master.cc:537] Master@127.3.107.189:41469 shutting down...
03:54:38.650 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.107.188:43667(127.3.107.188:43667)] connecting to peer
03:54:38.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.675733 3951 master.cc:552] Master@127.3.107.189:41469 shutdown complete.
03:54:38.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-tasksR1KPn/test-tmp/mini-kudu-cluster3865543798487354966/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:54:38.711 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:38.713 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:38.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.768008 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:38.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:38.768335 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:38.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:38.773 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:38.782 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.3.107.188:43667(127.3.107.188:43667)] Successfully connected to peer
03:54:38.785 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:38.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.786470 3785 master_runner.cc:186] Error getting master registration for 127.3.107.190:38431: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111),
03:54:38.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:38.786902 3785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:38.838 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:38.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:38.839 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:38.855 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:54:38.863 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.3.107.188
03:54:39.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:39 dist-test-slave-j019 krb5kdc[3515](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567671, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.3.107.188@KRBTEST.COM
03:54:39.083 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xf2e79d9c, L:/127.0.0.1:55308 - R:/127.3.107.188:43667] using SASL/GSSAPI
03:54:39.098 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.122 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.126 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.127 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.139 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.141 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.142 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.142 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.144 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.146 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.146 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.147 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.148 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.160 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.162 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.165 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.165 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.168 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.180 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.183 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.184 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.185 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.185 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.186 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.186 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.199 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.202 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.202 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.201 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.203 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.205 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.205 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.206 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.207 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.220 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.222 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.222 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.222 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.223 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.226 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.226 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.300 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.302 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.303 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.303 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.304 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.306 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.307 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.382 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.382 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.382 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.383 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.385 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.385 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.700 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.703 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.703 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.704 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.706 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.707 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.707 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:39.771284 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:39.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:39.771538 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:39.799 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.802 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.802 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.802 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.803 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.804 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.804 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.804 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.804 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:39.900 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:39.903 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:39.903 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.903 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.904 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:39.906 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:39.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:40.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:40.774567 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:40.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:40.774839 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:40.940 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:40.941 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:40.942 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:40.942 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:40.943 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:40.944 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:40.945 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:40.945 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:40.945 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:41.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:41.777568 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:41.777808 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:42.609156 4037 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:42.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:42.610128 4037 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:42.613018 4037 leader_election.cc:288] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c89b68037d064339841d1029aa535857 (127.3.107.190:38431), 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469)
03:54:42.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:42.616082 3807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111) [suppressed 5 similar messages]
03:54:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:42.619930 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c89b68037d064339841d1029aa535857 (127.3.107.190:38431): Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:42.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:42.620625 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469): Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:42.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:42.620891 3807 leader_election.cc:302] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a98d682389e467b8533c5e05812bae9; no voters: 6f3c8f0a15614ee398009cc1a1cf6645, c89b68037d064339841d1029aa535857
03:54:42.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:42.621454 4037 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:54:42.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:42.781353 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:42.781643 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:43.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:43.402 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:43.402 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:43.403 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:43.403 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:43.404 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:43.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:43.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:43.405 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:43.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:43 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567683, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:43.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:43.454221 3799 init.cc:295] Successfully reacquired a new kerberos TGT
03:54:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:43.784345 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:43.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:43.784664 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:43.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:43.789559 3785 master_runner.cc:186] Error getting master registration for 127.3.107.190:38431: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111),
03:54:43.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:43.789934 3785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:44.600 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:44.601 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:44.602 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:44.602 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:44.603 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:44.604 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:44.604 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:44.604 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:44.605 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:44.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:44.787402 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:44.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:44.787654 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:45.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:45.790611 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:45.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:45.790928 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:46.794001 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:46.794247 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:47.796998 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:47.797358 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:48.060 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:48.062 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:48.063 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:48.063 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:48.064 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:48.065 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:48.065 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:48.066 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:48.066 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:48.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:48.792374 3785 master_runner.cc:186] Error getting master registration for 127.3.107.190:38431: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111),
03:54:48.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:48.792840 3785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:48.800139 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:48.800372 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:49.672811 4049 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:49.673221 4049 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:49.675052 4049 leader_election.cc:288] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c89b68037d064339841d1029aa535857 (127.3.107.190:38431), 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469)
03:54:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:49.677517 3807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:54:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:49.680171 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c89b68037d064339841d1029aa535857 (127.3.107.190:38431): Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:49.680857 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469): Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:49.681116 3807 leader_election.cc:302] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a98d682389e467b8533c5e05812bae9; no voters: 6f3c8f0a15614ee398009cc1a1cf6645, c89b68037d064339841d1029aa535857
03:54:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:49.681681 4049 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:54:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:49.802987 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:49.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:49.803232 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:50.806025 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:50.806288 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:50.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:50.846164 3004 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0720 after lost signal to thread 2936
03:54:50.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:50.846773 3004 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba620 after lost signal to thread 3007
03:54:51.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:51.269986 4055 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:51.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:51.270434 4055 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:51.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:51.272090 4055 leader_election.cc:288] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c89b68037d064339841d1029aa535857 (127.3.107.190:38431), 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469)
03:54:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:51.277366 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c89b68037d064339841d1029aa535857 (127.3.107.190:38431): Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:51.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:51.278036 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469): Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:51.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:51.278421 3807 leader_election.cc:302] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a98d682389e467b8533c5e05812bae9; no voters: 6f3c8f0a15614ee398009cc1a1cf6645, c89b68037d064339841d1029aa535857
03:54:51.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:51.279213 4055 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:54:51.620 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:51.621 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:51.623 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:51.623 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:51.624 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:51.626 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:51.626 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:51.626 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:51.626 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:51.809142 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:51.809428 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:52.813037 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:52.813364 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:53.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:54:53 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567693, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:54:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:53.456851 3799 init.cc:295] Successfully reacquired a new kerberos TGT
03:54:53.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:53.796134 3785 master_runner.cc:186] Error getting master registration for 127.3.107.190:38431: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111),
03:54:53.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:53.796607 3785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:53.816622 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:53.816870 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:54.819573 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:54.819834 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:55.320 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:55.321 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:55.322 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:55.322 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:55.322 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:55.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:55.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:55.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:55.325 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:55.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:55.822623 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:55.822868 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:56.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:56.481585 4068 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:54:56.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:56.482120 4068 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c89b68037d064339841d1029aa535857" member_type: VOTER last_known_addr { host: "127.3.107.190" port: 38431 } } peers { permanent_uuid: "6f3c8f0a15614ee398009cc1a1cf6645" member_type: VOTER last_known_addr { host: "127.3.107.189" port: 41469 } } peers { permanent_uuid: "4a98d682389e467b8533c5e05812bae9" member_type: VOTER last_known_addr { host: "127.3.107.188" port: 43667 } }
03:54:56.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:56.483870 4068 leader_election.cc:288] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c89b68037d064339841d1029aa535857 (127.3.107.190:38431), 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469)
03:54:56.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:56.486521 3807 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111) [suppressed 3 similar messages]
03:54:56.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:56.489542 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c89b68037d064339841d1029aa535857 (127.3.107.190:38431): Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111)
03:54:56.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:56.490132 3807 leader_election.cc:334] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6f3c8f0a15614ee398009cc1a1cf6645 (127.3.107.189:41469): Network error: Client connection negotiation failed: client connection to 127.3.107.189:41469: connect: Connection refused (error 111)
03:54:56.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:56.490386 3807 leader_election.cc:302] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4a98d682389e467b8533c5e05812bae9; no voters: 6f3c8f0a15614ee398009cc1a1cf6645, c89b68037d064339841d1029aa535857
03:54:56.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:56.490902 4068 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:54:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:56.825829 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:56.826097 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:57.829001 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:57.829252 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:58.360 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:54:58.362 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:54:58.362 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:58.363 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:58.364 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:54:58.365 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:58.365 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:58.365 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:54:58.365 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:54:58.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:58.799538 3785 master_runner.cc:186] Error getting master registration for 127.3.107.190:38431: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111),
03:54:58.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:54:58.799916 3785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:54:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:58.832171 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:58.832422 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:54:59.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:59.835276 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:54:59.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:54:59.835516 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:00.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:00.707502 2850 debug-util.cc:398] Leaking SignalData structure 0x7b0800082360 after lost signal to thread 2787
03:55:00.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:00.708621 2850 debug-util.cc:398] Leaking SignalData structure 0x7b0800067e60 after lost signal to thread 2853
03:55:00.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:00.838335 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:00.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:00.838572 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:01.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:01.841652 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:01.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:01.841929 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:02.040 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:55:02.041 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:55:02.047 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:02.047 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:02.048 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:02.048 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:02.048 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:02.048 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:02.049 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:55:02.180 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:55:02.182 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:02.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:02.182 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:02.183 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:55:02.184 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:02.184 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:02.184 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:02.185 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:55:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:02.845554 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:02.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:02.845901 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:03.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 03:55:03 dist-test-slave-j019 krb5kdc[3515](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790567703, etypes {rep=17 tkt=17 ses=17}, kudu/127.3.107.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:55:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:03.459219 3799 init.cc:295] Successfully reacquired a new kerberos TGT
03:55:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:03.803004 3785 master_runner.cc:186] Error getting master registration for 127.3.107.190:38431: Network error: Client connection negotiation failed: client connection to 127.3.107.190:38431: connect: Connection refused (error 111),
03:55:03.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:03.803380 3785 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:55:03.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:03.848490 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:03.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:03.848778 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:04.851068 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:04.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:04.851310 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:05.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:05.853755 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:05.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:05.854000 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:05.860 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:55:05.861 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:55:05.861 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:05.861 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:05.861 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:05.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:05.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:05.862 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:05.864 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:55:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:06.856395 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:06.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:06.856698 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:07.219 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:55:07.220 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:55:07.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:07.221 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:07.221 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:07.221 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:07.221 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:07.221 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:07.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:55:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:07.858963 3872 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4a98d682389e467b8533c5e05812bae9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:55:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:07.859274 3872 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:55:08.139 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.190:38431(127.3.107.190:38431)] connecting to peer
03:55:08.140 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.3.107.189:41469(127.3.107.189:41469)] connecting to peer
03:55:08.141 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:08.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.3.107.190:38431(127.3.107.190:38431)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:08.141 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.190:38431: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431
03:55:08.141 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:08.141 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.189:41469(127.3.107.189:41469)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:08.142 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.3.107.189:41469: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
03:55:08.143 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469; will retry
03:55:08.146 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29545), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [237ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [493ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [496ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [524ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [525ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [528ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [529ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [540ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [543ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [546ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [563ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [565ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [585ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [620ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [701ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [706ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [782ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [784ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1097ms] refreshing cache from master, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1104ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1106ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1197ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1202ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1305ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [2337ms] refreshing cache from master, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [2339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [2341ms] too many traces: truncated at 100 traces, deferred=Deferred@671203223(state=PENDING, result=null, callback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272), errback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272)))
03:55:08.151 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.107.190:38431(127.3.107.190:38431)] disconnecting while in state TERMINATED
03:55:08.152 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.107.188:43667(127.3.107.188:43667)] disconnecting while in state READY
03:55:08.152 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.3.107.189:41469(127.3.107.189:41469)] disconnecting while in state TERMINATED
03:55:08.156 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:55:08.161 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:55:08.163 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.3.107.188:43667(127.3.107.188:43667)] handling channelInactive
03:55:08.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.3.107.188:43667(127.3.107.188:43667)] cleaning up while in state READY due to: connection closed
03:55:08.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:08.174963 3502 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 3785
03:55:09.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:55:09Z chronyd exiting
03:55:10.390 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:55:10.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:10.392104 2774 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 3057
03:55:10.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:10.421954 2774 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 3216
03:55:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:10.445012 3004 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac1e0 after lost signal to thread 2936
03:55:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 03:55:10.445693 3004 debug-util.cc:398] Leaking SignalData structure 0x7b08000a0c00 after lost signal to thread 3007
03:55:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:10.452694 2774 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 3359
03:55:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:10.481640 2774 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 2786
03:55:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:10.513616 2774 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 2856
03:55:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 03:55:10.542626 2774 external_mini_cluster.cc:1595] Killing /tmp/dist-test-tasksR1KPn/build/tsan/bin/kudu with pid 2931
03:55:11.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T03:55:11Z chronyd exiting
03:55:12.758 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29545), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [237ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [493ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [496ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [524ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [525ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [528ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [529ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [540ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [543ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [546ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [563ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [565ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [585ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [620ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [701ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [706ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [782ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [784ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1097ms] refreshing cache from master, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1104ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1106ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1197ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1202ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1305ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [2337ms] refreshing cache from master, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [2339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [2341ms] too many traces: truncated at 100 traces, deferred=Deferred@671203223(state=PENDING, result=null, callback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272), errback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.107.189:41469
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29545), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [237ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [493ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [496ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [524ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [525ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [528ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [529ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [540ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [543ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [546ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [563ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [565ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [585ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [620ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [701ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [706ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [782ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [784ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1097ms] refreshing cache from master, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1104ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1106ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1197ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1202ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1305ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [2337ms] refreshing cache from master, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [2339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [2341ms] too many traces: truncated at 100 traces, deferred=Deferred@671203223(state=PENDING, result=null, callback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272), errback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.107.189:41469
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
03:55:14.007 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:55:14.007 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29545), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [237ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [493ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [496ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [524ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [525ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [528ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [529ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [540ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [543ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [546ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [563ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [565ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [585ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [620ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [701ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [706ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [782ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [784ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1097ms] refreshing cache from master, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1104ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1106ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1197ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1202ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1305ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [2337ms] refreshing cache from master, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [2339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [2341ms] too many traces: truncated at 100 traces, deferred=Deferred@671203223(state=PENDING, result=null, callback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272), errback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.107.189:41469
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29545), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [181ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [237ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [493ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [496ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [524ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [525ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [528ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [529ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [540ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [543ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [544ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [546ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [563ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [565ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [566ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [567ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [577ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [581ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [584ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [585ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [596ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [600ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [601ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [603ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [605ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [620ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [623ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [624ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [701ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [703ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [705ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [706ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [777ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [780ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [782ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [783ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [784ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1097ms] refreshing cache from master, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1097ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1102ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1104ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1106ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1197ms] refreshing cache from master, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1200ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1201ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1202ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1202ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1297ms] refreshing cache from master, [1297ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [1298ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [1300ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [1302ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.190:38431: Network error: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431, [1303ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.188:43667: OK, [1304ms] Sub RPC ConnectToMaster: received response from server master-127.3.107.189:41469: Network error: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [1305ms] delaying RPC due to: Service unavailable: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469, [2337ms] refreshing cache from master, [2337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.188:43667, [2338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.190:38431, [2339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.3.107.189:41469, [2341ms] too many traces: truncated at 100 traces, deferred=Deferred@671203223(state=PENDING, result=null, callback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272), errback=(continuation of Deferred@981161618 after retry RPC after error@305238985) -> (continuation of Deferred@6684144 after retry RPC after error@624760684) -> (continuation of Deferred@1167202975 after retry RPC after error@1537435199) -> (continuation of Deferred@1954974406 after retry RPC after error@969766221) -> (continuation of Deferred@1818150409 after retry RPC after error@739034915) -> (continuation of Deferred@1822210270 after retry RPC after error@1042931485) -> (continuation of Deferred@618477663 after retry RPC after error@1847772728) -> (continuation of Deferred@1735514474 after retry RPC after error@399520686) -> (continuation of Deferred@1518530251 after retry RPC after error@585756837) -> (continuation of Deferred@1270531867 after retry RPC after error@1566691457) -> (continuation of Deferred@282630480 after retry RPC after error@2111385398) -> (continuation of Deferred@1615248777 after retry RPC after error@871402712) -> (continuation of Deferred@1731625455 after retry RPC after error@1394778784) -> (continuation of Deferred@115210168 after retry RPC after error@838610423) -> (continuation of Deferred@1654062046 after retry RPC after error@619147918) -> (continuation of Deferred@1299067293 after retry RPC after error@497892946) -> (continuation of Deferred@1219526194 after retry RPC after error@1465648808) -> (continuation of Deferred@1803691783 after retry RPC after error@531221387) -> (continuation of Deferred@1411610530 after retry RPC after error@1669157576) -> (continuation of Deferred@423492543 after retry RPC after error@1209634432) -> (continuation of Deferred@130580038 after retry RPC after error@246056805) -> (continuation of Deferred@2017228044 after retry RPC after error@2094369029) -> (continuation of Deferred@950792855 after retry RPC after error@1658899272)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.3.107.188:43667,127.3.107.190:38431,127.3.107.189:41469) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.190:38431(127.3.107.190:38431): Connection refused: /127.3.107.190:38431,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.3.107.189:41469(127.3.107.189:41469): Connection refused: /127.3.107.189:41469
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.3.107.189:41469
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)