02:37:35.959 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:37:35.975 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskvhYo2y/test-tmp
02:37:35.978 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:37:35.982 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskvhYo2y/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:37:36.209 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:37:36.366 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:37:36.367 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:37:36.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.372148 374 mini_oidc.cc:152] Starting JWKS server
02:37:36.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.376330 374 webserver.cc:460] Webserver started at http://127.0.0.1:40505/ using document root <none> and password file <none>
02:37:36.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.376830 374 mini_oidc.cc:189] Starting OIDC Discovery server
02:37:36.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.379024 374 webserver.cc:460] Webserver started at http://127.0.0.1:37661/ using document root <none> and password file <none>
02:37:36.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:37:36Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:37:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:37:36Z Disabled control of system clock
02:37:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.429648 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/wal
02:37:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data
02:37:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:36.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/logs
02:37:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data/info.pb
02:37:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:42771
02:37:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
02:37:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:36.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:36.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40505/jwks.json with env {}
02:37:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.738708 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.739311 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:36.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.739943 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.772684 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.772976 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.773164 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:36.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.773401 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:36.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.773669 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:36.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.773860 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40505/jwks.json
02:37:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.808547 386 master_runner.cc:386] Master server non-default flags:
02:37:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:36.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:36.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data
02:37:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/wal
02:37:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:42771
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40505/jwks.json
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data/info.pb
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
02:37:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/logs
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:36.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.809805 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.811443 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:36.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.831525 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.831516 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:36.833472 386 server_base.cc:1034] running on GCE node
02:37:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:36.831590 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.028527 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:38.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.031136 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:38.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.032469 386 hybrid_clock.cc:648] HybridClock initialized: now 1790649458032427 us; error 68 us; skew 500 ppm
02:37:38.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.033248 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:38.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.043790 386 webserver.cc:460] Webserver started at http://127.0.93.190:46069/ using document root <none> and password file <none>
02:37:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.044764 386 fs_manager.cc:359] Metadata directory not provided
02:37:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.044945 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:38.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.045316 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.050139 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data/instance:
02:37:38.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24934d79cdf149f3a6f0786e2fe9d39c"
02:37:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:38 on dist-test-slave-chs2"
02:37:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.051335 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/wal/instance:
02:37:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24934d79cdf149f3a6f0786e2fe9d39c"
02:37:38.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:38 on dist-test-slave-chs2"
02:37:38.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.058795 386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.010s sys 0.000s
02:37:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.064087 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.065276 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:37:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.065548 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/wal
02:37:38.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "24934d79cdf149f3a6f0786e2fe9d39c"
02:37:38.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:38 on dist-test-slave-chs2"
02:37:38.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.065837 386 fs_report.cc:383] FS layout report
02:37:38.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:38.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/wal
02:37:38.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/wal
02:37:38.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data/data
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:38.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:38.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.102741 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:38.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.104174 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:38.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.104635 386 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.182605 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:42771
02:37:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.182678 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:42771 every 8 connection(s)
02:37:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.185447 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/data/info.pb
02:37:38.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.191123 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:37:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.193601 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 386
02:37:38.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.194525 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-0/wal/instance
02:37:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.199688 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/wal
02:37:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data
02:37:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/logs
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data/info.pb
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:34179
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:38.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40505/jwks.json with env {}
02:37:38.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.207654 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } has no permanent_uuid. Determining permanent_uuid...
02:37:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.236660 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } has no permanent_uuid. Determining permanent_uuid...
02:37:38.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.239194 406 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:38.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.243393 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34179: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.292939 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } attempt: 1
02:37:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.297223 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34179: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.365682 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } attempt: 2
02:37:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.369921 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34179: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:38.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.461398 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } attempt: 3
02:37:38.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.465402 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34179: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.529525 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.530059 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:38.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.530596 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.562668 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.562932 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.563179 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.563359 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.563568 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.563745 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40505/jwks.json
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.599788 456 master_runner.cc:386] Master server non-default flags:
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/wal
02:37:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:34179
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40505/jwks.json
02:37:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data/info.pb
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/logs
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.601143 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.603168 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:38.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.608810 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } attempt: 4
02:37:38.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.614282 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34179: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:38.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.624959 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:38.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:38.913715 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } attempt: 5
02:37:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.920698 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34179: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:39.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.468158 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } attempt: 6
02:37:39.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:39.475835 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:34179: Network error: Client connection negotiation failed: client connection to 127.0.93.189:34179: connect: Connection refused (error 111)
02:37:39.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.625020 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:39.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:38.625309 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:39.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:39.837127 467 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:37:39.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.837307 456 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:37:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.841056 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:39.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.843993 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:39.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.845449 456 hybrid_clock.cc:648] HybridClock initialized: now 1790649459845418 us; error 60 us; skew 500 ppm
02:37:39.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.846563 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.854329 456 webserver.cc:460] Webserver started at http://127.0.93.189:38051/ using document root <none> and password file <none>
02:37:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.855320 456 fs_manager.cc:359] Metadata directory not provided
02:37:39.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.855579 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:39.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.856120 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.861975 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data/instance:
02:37:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e88ec665bff4b33a590325a8244e1bb"
02:37:39.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:39 on dist-test-slave-chs2"
02:37:39.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.863471 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/wal/instance:
02:37:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e88ec665bff4b33a590325a8244e1bb"
02:37:39.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:39 on dist-test-slave-chs2"
02:37:39.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.871430 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
02:37:39.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.877244 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:39.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.878422 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:37:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.878697 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/wal
02:37:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e88ec665bff4b33a590325a8244e1bb"
02:37:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:39 on dist-test-slave-chs2"
02:37:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.878979 456 fs_report.cc:383] FS layout report
02:37:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:39.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/wal
02:37:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/wal
02:37:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data/data
02:37:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:39.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.960427 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.961838 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:39.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:39.962229 456 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:40.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.037983 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:34179
02:37:40.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.038057 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:34179 every 8 connection(s)
02:37:40.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.040555 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/data/info.pb
02:37:40.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.045964 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:37:40.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.045953 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 456
02:37:40.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.046360 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-1/wal/instance
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.048616 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/wal
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:40.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/logs
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data/info.pb
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:42181
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:40.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40505/jwks.json with env {}
02:37:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.060501 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } has no permanent_uuid. Determining permanent_uuid...
02:37:40.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.086037 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } has no permanent_uuid. Determining permanent_uuid...
02:37:40.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.103677 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } has no permanent_uuid. Determining permanent_uuid...
02:37:40.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.106927 479 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.111079 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.160585 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 1
02:37:40.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.165354 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.233812 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 2
02:37:40.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.237882 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.329344 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 3
02:37:40.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.333173 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.374684 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.375198 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.375644 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.406068 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.406379 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.406572 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.406761 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.407002 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.407212 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40505/jwks.json
02:37:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.441217 531 master_runner.cc:386] Master server non-default flags:
02:37:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:40.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/wal
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:42181
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40505/jwks.json
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data/info.pb
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/logs
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.442428 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.444123 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:40.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.466790 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.476614 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 4
02:37:40.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.481943 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.536276 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } attempt: 7
02:37:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.558743 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } has no permanent_uuid. Determining permanent_uuid...
02:37:40.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.567243 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.625835 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 1
02:37:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.631568 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.713140 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 2
02:37:40.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.719480 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.781389 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 5
02:37:40.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.787643 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.805080 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 3
02:37:40.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.811764 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:40.952417 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 4
02:37:40.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.958722 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:41.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.242314 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 5
02:37:41.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:41.248184 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:41.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.335253 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 6
02:37:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:41.347281 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.472236 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:40.467229 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:41.700073 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1234 milliseconds
02:37:41.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.700242 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:37:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.701642 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:41.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.704186 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.705554 531 hybrid_clock.cc:648] HybridClock initialized: now 1790649461705526 us; error 37 us; skew 500 ppm
02:37:41.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.706411 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:41.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.712993 531 webserver.cc:460] Webserver started at http://127.0.93.188:35371/ using document root <none> and password file <none>
02:37:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.714123 531 fs_manager.cc:359] Metadata directory not provided
02:37:41.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.714391 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:41.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.714923 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.720250 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data/instance:
02:37:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e48b648f17a54bf69ccafa0c0adc2844"
02:37:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:41 on dist-test-slave-chs2"
02:37:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.721467 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/wal/instance:
02:37:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e48b648f17a54bf69ccafa0c0adc2844"
02:37:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:41 on dist-test-slave-chs2"
02:37:41.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.728597 531 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
02:37:41.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.734318 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:41.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.735318 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.735667 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/wal
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e48b648f17a54bf69ccafa0c0adc2844"
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:41 on dist-test-slave-chs2"
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.736123 531 fs_report.cc:383] FS layout report
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/wal
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/wal
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data/data
02:37:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:41.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.776136 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:41.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.777645 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.778172 531 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:41.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.800748 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 6
02:37:41.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:41.819236 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:42181: Network error: Client connection negotiation failed: client connection to 127.0.93.188:42181: connect: Connection refused (error 111)
02:37:41.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.859161 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:42181
02:37:41.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.859236 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:42181 every 8 connection(s)
02:37:41.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.861701 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/data/info.pb
02:37:41.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.866859 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 531
02:37:41.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.867304 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/master-2/wal/instance
02:37:41.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.868198 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:37:41.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.885726 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } has no permanent_uuid. Determining permanent_uuid...
02:37:41.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.910887 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } has no permanent_uuid. Determining permanent_uuid...
02:37:41.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.922645 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } has no permanent_uuid. Determining permanent_uuid...
02:37:41.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.939180 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844: Bootstrap starting.
02:37:41.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.946134 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844: Neither blocks nor log segments found. Creating new log.
02:37:41.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.947576 609 log.cc:824] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844: Log is configured to *not* fsync() on all Append() calls
02:37:41.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.952291 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844: No bootstrap required, opened a new log
02:37:41.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.972968 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:41.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.973527 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:37:41.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.973714 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e48b648f17a54bf69ccafa0c0adc2844, State: Initialized, Role: FOLLOWER
02:37:41.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.974363 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:41.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.977048 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:41.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.977684 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: This master's current role is: FOLLOWER
02:37:41.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.978711 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: configured and running, proceeding with master startup.
02:37:42.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:42.003290 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:37:42.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:42.003633 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:37:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:41.999629 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:42771: OK, instance_id { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" instance_seqno: 1790649458161139 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:37:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.033260 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:37:42.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:42.033444 561 tablet.cc:2369] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:37:42.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.322505 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:37:42.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.322928 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:42.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.325095 618 leader_election.cc:288] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 24934d79cdf149f3a6f0786e2fe9d39c (127.0.93.190:42771), 1e88ec665bff4b33a590325a8244e1bb (127.0.93.189:34179)
02:37:42.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.407840 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 7
02:37:42.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.432703 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: Bootstrap starting.
02:37:42.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.439455 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: Neither blocks nor log segments found. Creating new log.
02:37:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.441998 529 log.cc:824] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: Log is configured to *not* fsync() on all Append() calls
02:37:42.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.447583 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: No bootstrap required, opened a new log
02:37:42.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.477862 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.478754 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:37:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.479039 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1e88ec665bff4b33a590325a8244e1bb, State: Initialized, Role: FOLLOWER
02:37:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.480058 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:42.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.482739 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:42.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.483896 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [sys.catalog]: This master's current role is: FOLLOWER
02:37:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.484992 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [sys.catalog]: configured and running, proceeding with master startup.
02:37:42.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.485168 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e48b648f17a54bf69ccafa0c0adc2844" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1e88ec665bff4b33a590325a8244e1bb" is_pre_election: true
02:37:42.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.486068 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e48b648f17a54bf69ccafa0c0adc2844 in term 0.
02:37:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.487496 559 leader_election.cc:302] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1e88ec665bff4b33a590325a8244e1bb, e48b648f17a54bf69ccafa0c0adc2844; no voters:
02:37:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.489410 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:37:42.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.489885 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:37:42.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.490316 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 0 FOLLOWER]: Advancing to term 1
02:37:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.501124 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.505728 618 leader_election.cc:288] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [CANDIDATE]: Term 1 election: Requested vote from peers 24934d79cdf149f3a6f0786e2fe9d39c (127.0.93.190:42771), 1e88ec665bff4b33a590325a8244e1bb (127.0.93.189:34179)
02:37:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.506269 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e48b648f17a54bf69ccafa0c0adc2844" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1e88ec665bff4b33a590325a8244e1bb"
02:37:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.506814 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [term 0 FOLLOWER]: Advancing to term 1
02:37:42.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.516665 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e48b648f17a54bf69ccafa0c0adc2844 in term 1.
02:37:42.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.518409 559 leader_election.cc:302] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1e88ec665bff4b33a590325a8244e1bb, e48b648f17a54bf69ccafa0c0adc2844; no voters:
02:37:42.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.522075 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 1 FOLLOWER]: Leader election won for term 1
02:37:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.528162 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [term 1 LEADER]: Becoming Leader. State: Replica: e48b648f17a54bf69ccafa0c0adc2844, State: Running, Role: LEADER
02:37:42.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.529171 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:42.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.542788 638 sys_catalog.cc:455] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e48b648f17a54bf69ccafa0c0adc2844. Latest consensus state: current_term: 1 leader_uuid: "e48b648f17a54bf69ccafa0c0adc2844" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:42.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.544435 638 sys_catalog.cc:458] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: This master's current role is: LEADER
02:37:42.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.548007 643 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
02:37:42.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.560367 643 catalog_manager.cc:1484] Initializing Kudu cluster ID...
02:37:42.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.585444 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [term 1 FOLLOWER]: Refusing update from remote peer e48b648f17a54bf69ccafa0c0adc2844: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:37:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.587996 638 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:37:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:42.601403 648 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:37:42.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:42.602043 648 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:37:42.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.577991 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:42771: OK, instance_id { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" instance_seqno: 1790649458161139 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:37:42.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.629199 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:37:42.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.643868 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [sys.catalog]: SysCatalogTable state changed. Reason: New leader e48b648f17a54bf69ccafa0c0adc2844. Latest consensus state: current_term: 1 leader_uuid: "e48b648f17a54bf69ccafa0c0adc2844" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.646019 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [sys.catalog]: This master's current role is: FOLLOWER
02:37:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.653043 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e48b648f17a54bf69ccafa0c0adc2844" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.654093 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: This master's current role is: LEADER
02:37:42.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.657742 643 catalog_manager.cc:1347] Generated new cluster ID: 942a728a32a64a049460cf6b156c6408
02:37:42.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.657997 643 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
02:37:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.682267 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e48b648f17a54bf69ccafa0c0adc2844" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.682983 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb [sys.catalog]: This master's current role is: FOLLOWER
02:37:42.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.706341 643 catalog_manager.cc:1370] Generated new certificate authority record
02:37:42.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.713055 643 catalog_manager.cc:1504] Loading token signing keys...
02:37:42.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.771873 643 catalog_manager.cc:5868] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844: Generated new TSK 0
02:37:42.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.775018 643 catalog_manager.cc:1514] Initializing in-progress tserver states...
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.816546 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/wal
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/logs
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data/info.pb
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:42.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:42.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:37:42.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.852613 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } attempt: 7
02:37:42.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.896584 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: Bootstrap starting.
02:37:42.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.915831 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: Neither blocks nor log segments found. Creating new log.
02:37:42.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.921455 454 log.cc:824] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: Log is configured to *not* fsync() on all Append() calls
02:37:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:42.946862 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: No bootstrap required, opened a new log
02:37:43.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.026171 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.029007 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:37:43.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.030030 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 24934d79cdf149f3a6f0786e2fe9d39c, State: Initialized, Role: FOLLOWER
02:37:43.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.032481 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.053702 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e48b648f17a54bf69ccafa0c0adc2844" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "24934d79cdf149f3a6f0786e2fe9d39c"
02:37:43.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.056143 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e48b648f17a54bf69ccafa0c0adc2844" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" is_pre_election: true
02:37:43.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.057839 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 0 FOLLOWER]: Advancing to term 1
02:37:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.069783 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 1 FOLLOWER]: Refusing update from remote peer e48b648f17a54bf69ccafa0c0adc2844: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:37:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.074802 638 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [LEADER]: Connected to new peer: Peer: permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:37:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.075944 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [sys.catalog]: configured and running, proceeding with master startup.
02:37:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.075484 658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.081013 658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [sys.catalog]: This master's current role is: FOLLOWER
02:37:43.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.115137 408 tablet.cc:2369] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:37:43.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.164089 658 raft_consensus.cc:482] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 1 FOLLOWER]: Starting pre-election (detected failure of leader e48b648f17a54bf69ccafa0c0adc2844)
02:37:43.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.164582 658 raft_consensus.cc:504] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } }
02:37:43.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.171144 503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "1e88ec665bff4b33a590325a8244e1bb" is_pre_election: true
02:37:43.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.171736 661 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e48b648f17a54bf69ccafa0c0adc2844" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:43.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.172544 661 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [sys.catalog]: This master's current role is: FOLLOWER
02:37:43.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.175051 584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "e48b648f17a54bf69ccafa0c0adc2844" is_pre_election: true
02:37:43.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.177083 404 leader_election.cc:302] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 24934d79cdf149f3a6f0786e2fe9d39c; no voters: 1e88ec665bff4b33a590325a8244e1bb, e48b648f17a54bf69ccafa0c0adc2844
02:37:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.178131 661 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:37:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.179778 658 leader_election.cc:288] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 1e88ec665bff4b33a590325a8244e1bb (127.0.93.189:34179), e48b648f17a54bf69ccafa0c0adc2844 (127.0.93.188:42181)
02:37:43.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.179010 659 sys_catalog.cc:455] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [sys.catalog]: SysCatalogTable state changed. Reason: New leader e48b648f17a54bf69ccafa0c0adc2844. Latest consensus state: current_term: 1 leader_uuid: "e48b648f17a54bf69ccafa0c0adc2844" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:43.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.181929 659 sys_catalog.cc:458] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c [sys.catalog]: This master's current role is: FOLLOWER
02:37:43.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.221668 676 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:37:43.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.332231 655 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.332863 655 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:43.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.333619 655 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:43.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.386993 655 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:43.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.388747 655 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.448897 655 tablet_server_runner.cc:78] Tablet server non-default flags:
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/wal
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data/info.pb
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/logs
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:43.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.450353 655 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:43.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.452364 655 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.477382 682 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.527330 638 sys_catalog.cc:455] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e48b648f17a54bf69ccafa0c0adc2844" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "24934d79cdf149f3a6f0786e2fe9d39c" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 42771 } } peers { permanent_uuid: "1e88ec665bff4b33a590325a8244e1bb" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 34179 } } peers { permanent_uuid: "e48b648f17a54bf69ccafa0c0adc2844" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 42181 } } }
02:37:43.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.528669 638 sys_catalog.cc:458] T 00000000000000000000000000000000 P e48b648f17a54bf69ccafa0c0adc2844 [sys.catalog]: This master's current role is: LEADER
02:37:43.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.609853 648 catalog_manager.cc:1259] Loaded cluster ID: 942a728a32a64a049460cf6b156c6408
02:37:43.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.610756 648 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: loading cluster ID for follower catalog manager: success
02:37:43.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.633347 648 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: acquiring CA information for follower catalog manager: success
02:37:43.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:43.649426 648 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1e88ec665bff4b33a590325a8244e1bb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:37:44.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:44.243409 676 catalog_manager.cc:1259] Loaded cluster ID: 942a728a32a64a049460cf6b156c6408
02:37:44.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:44.244431 676 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: loading cluster ID for follower catalog manager: success
02:37:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:44.270278 676 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: acquiring CA information for follower catalog manager: success
02:37:44.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:44.301507 676 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 24934d79cdf149f3a6f0786e2fe9d39c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:37:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:44.868126 681 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 655
02:37:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:45.036885 681 kernel_stack_watchdog.cc:198] Thread 655 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 387ms:
02:37:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:37:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:37:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:37:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:37:45.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:43.477885 683 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:45.046650 655 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.568s user 0.002s sys 0.000s
02:37:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:45.047039 655 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.568s user 0.002s sys 0.001s
02:37:45.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:45.054553 685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:45.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.056474 655 server_base.cc:1034] running on GCE node
02:37:45.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.058171 655 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:45.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.074744 655 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.078720 655 hybrid_clock.cc:648] HybridClock initialized: now 1790649465078688 us; error 57 us; skew 500 ppm
02:37:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.079826 655 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.106228 655 webserver.cc:460] Webserver started at http://127.0.93.129:36701/ using document root <none> and password file <none>
02:37:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.107795 655 fs_manager.cc:359] Metadata directory not provided
02:37:45.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.108285 655 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:45.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.108938 655 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:45.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.116498 655 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data/instance:
02:37:45.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15a2f9a4399247379536823ab11d8573"
02:37:45.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:45 on dist-test-slave-chs2"
02:37:45.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.118274 655 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/wal/instance:
02:37:45.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15a2f9a4399247379536823ab11d8573"
02:37:45.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:45 on dist-test-slave-chs2"
02:37:45.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.135867 655 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.006s sys 0.006s
02:37:45.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.150044 695 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.151810 655 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.005s sys 0.003s
02:37:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.152424 655 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/wal
02:37:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "15a2f9a4399247379536823ab11d8573"
02:37:45.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:45 on dist-test-slave-chs2"
02:37:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.153095 655 fs_report.cc:383] FS layout report
02:37:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/wal
02:37:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/wal
02:37:45.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data/data
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:45.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.243448 655 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:45.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.245471 655 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:45.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.246016 655 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:45.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.250433 655 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:37:45.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.256757 655 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:37:45.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.257016 655 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:37:45.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.257702 655 ts_tablet_manager.cc:610] Registered 0 tablets
02:37:45.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.257931 655 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:37:45.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.480942 655 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:39783
02:37:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.481238 807 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:39783 every 8 connection(s)
02:37:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.484759 655 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/data/info.pb
02:37:45.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.492794 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 655
02:37:45.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.493328 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-0/wal/instance
02:37:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.537995 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/wal
02:37:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data
02:37:45.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/logs
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data/info.pb
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:45.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:45.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:37:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.557912 808 heartbeater.cc:347] Connected to a master server at 127.0.93.188:42181
02:37:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.558382 808 heartbeater.cc:464] Registering TS with master...
02:37:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.559508 808 heartbeater.cc:511] Master 127.0.93.188:42181 requested a full tablet report, sending...
02:37:45.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.561132 809 heartbeater.cc:347] Connected to a master server at 127.0.93.189:34179
02:37:45.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.561448 809 heartbeater.cc:464] Registering TS with master...
02:37:45.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.562178 809 heartbeater.cc:511] Master 127.0.93.189:34179 requested a full tablet report, sending...
02:37:45.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.562247 573 ts_manager.cc:194] Registered new tserver with Master: 15a2f9a4399247379536823ab11d8573 (127.0.93.129:39783)
02:37:45.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.565728 494 ts_manager.cc:194] Registered new tserver with Master: 15a2f9a4399247379536823ab11d8573 (127.0.93.129:39783)
02:37:45.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.569078 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:51025
02:37:45.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.574302 812 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42771
02:37:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.574648 812 heartbeater.cc:464] Registering TS with master...
02:37:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.575307 812 heartbeater.cc:511] Master 127.0.93.190:42771 requested a full tablet report, sending...
02:37:45.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:45.580070 419 ts_manager.cc:194] Registered new tserver with Master: 15a2f9a4399247379536823ab11d8573 (127.0.93.129:39783)
02:37:46.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.019560 818 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:46.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.020238 818 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:46.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.020977 818 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:46.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.083117 818 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:46.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.084494 818 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
02:37:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:46.142122 818 tablet_server_runner.cc:78] Tablet server non-default flags:
02:37:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data
02:37:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/wal
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data/info.pb
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/logs
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:46.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:46.143401 818 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:46.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:46.145426 818 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:46.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.169034 824 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:46.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:46.573891 808 heartbeater.cc:503] Master 127.0.93.188:42181 was elected leader, sending a full tablet report...
02:37:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.170011 825 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:46.170989 818 server_base.cc:1034] running on GCE node
02:37:47.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:46.172227 827 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:47.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.403874 818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:47.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.406665 818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:47.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.408134 818 hybrid_clock.cc:648] HybridClock initialized: now 1790649467408090 us; error 51 us; skew 500 ppm
02:37:47.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.408875 818 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:47.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.415429 818 webserver.cc:460] Webserver started at http://127.0.93.130:42935/ using document root <none> and password file <none>
02:37:47.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.416548 818 fs_manager.cc:359] Metadata directory not provided
02:37:47.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.416738 818 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:47.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.417117 818 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:47.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.422152 818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data/instance:
02:37:47.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3cd7c010a2f149a1ae0c10f6cbd7eebd"
02:37:47.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:47 on dist-test-slave-chs2"
02:37:47.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.423200 818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/wal/instance:
02:37:47.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3cd7c010a2f149a1ae0c10f6cbd7eebd"
02:37:47.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:47 on dist-test-slave-chs2"
02:37:47.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.431885 818 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
02:37:47.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.437202 836 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.438326 818 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
02:37:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.438608 818 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/wal
02:37:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3cd7c010a2f149a1ae0c10f6cbd7eebd"
02:37:47.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:47 on dist-test-slave-chs2"
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.438880 818 fs_report.cc:383] FS layout report
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/wal
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/wal
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data/data
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:47.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:47.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.489611 818 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:47.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.491492 818 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:47.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.492063 818 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:47.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.495195 818 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:37:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.499642 818 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:37:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.499830 818 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:37:47.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.500099 818 ts_tablet_manager.cc:610] Registered 0 tablets
02:37:47.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.500252 818 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:37:47.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.652573 818 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:37349
02:37:47.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.652666 948 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:37349 every 8 connection(s)
02:37:47.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.654953 818 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/data/info.pb
02:37:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.655720 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 818
02:37:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.656379 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-1/wal/instance
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.675694 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/wal
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:47.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/logs
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data/info.pb
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:37:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.710906 949 heartbeater.cc:347] Connected to a master server at 127.0.93.188:42181
02:37:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.711381 949 heartbeater.cc:464] Registering TS with master...
02:37:47.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.712606 949 heartbeater.cc:511] Master 127.0.93.188:42181 requested a full tablet report, sending...
02:37:47.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.712989 950 heartbeater.cc:347] Connected to a master server at 127.0.93.189:34179
02:37:47.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.713322 950 heartbeater.cc:464] Registering TS with master...
02:37:47.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.714082 950 heartbeater.cc:511] Master 127.0.93.189:34179 requested a full tablet report, sending...
02:37:47.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.715234 573 ts_manager.cc:194] Registered new tserver with Master: 3cd7c010a2f149a1ae0c10f6cbd7eebd (127.0.93.130:37349)
02:37:47.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.716779 494 ts_manager.cc:194] Registered new tserver with Master: 3cd7c010a2f149a1ae0c10f6cbd7eebd (127.0.93.130:37349)
02:37:47.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.718185 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:42435
02:37:47.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.721895 953 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42771
02:37:47.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.722236 953 heartbeater.cc:464] Registering TS with master...
02:37:47.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.722903 953 heartbeater.cc:511] Master 127.0.93.190:42771 requested a full tablet report, sending...
02:37:47.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:47.725579 419 ts_manager.cc:194] Registered new tserver with Master: 3cd7c010a2f149a1ae0c10f6cbd7eebd (127.0.93.130:37349)
02:37:48.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.007560 956 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:48.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.008119 956 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:48.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.008576 956 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.038609 956 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:48.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.039749 956 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:48.075090 956 tablet_server_runner.cc:78] Tablet server non-default flags:
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:45441
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/wal
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data/info.pb
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:42771,127.0.93.189:34179,127.0.93.188:42181
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/logs
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:48.076256 956 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:48.077979 956 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.097786 965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:48.721959 949 heartbeater.cc:503] Master 127.0.93.188:42181 was elected leader, sending a full tablet report...
02:37:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.097963 966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:48.099668 956 server_base.cc:1034] running on GCE node
02:37:49.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:48.097983 968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:49.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.294220 956 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.296919 956 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.298352 956 hybrid_clock.cc:648] HybridClock initialized: now 1790649469298315 us; error 57 us; skew 500 ppm
02:37:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.299075 956 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.310176 956 webserver.cc:460] Webserver started at http://127.0.93.131:36979/ using document root <none> and password file <none>
02:37:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.311066 956 fs_manager.cc:359] Metadata directory not provided
02:37:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.311257 956 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:49.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.311614 956 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:49.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.316376 956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data/instance:
02:37:49.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbac8d84b06d4f9dbf2a9a1cff7bda2e"
02:37:49.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:49 on dist-test-slave-chs2"
02:37:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.317386 956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/wal/instance:
02:37:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbac8d84b06d4f9dbf2a9a1cff7bda2e"
02:37:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:49 on dist-test-slave-chs2"
02:37:49.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.324120 956 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
02:37:49.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.329874 977 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:49.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.331174 956 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
02:37:49.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.331571 956 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/wal
02:37:49.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbac8d84b06d4f9dbf2a9a1cff7bda2e"
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:49 on dist-test-slave-chs2"
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.331923 956 fs_report.cc:383] FS layout report
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/wal
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/wal
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data/data
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:49.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.376922 956 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:49.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.378422 956 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:49.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.378994 956 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:49.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.381642 956 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
02:37:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.386051 956 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:37:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.386276 956 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:37:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.386646 956 ts_tablet_manager.cc:610] Registered 0 tablets
02:37:49.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.386848 956 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:37:49.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.542589 956 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:35463
02:37:49.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.542668 1090 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:35463 every 8 connection(s)
02:37:49.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.545197 956 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/data/info.pb
02:37:49.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.547983 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 956
02:37:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.548480 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster8639714047116338305/ts-2/wal/instance
02:37:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.602169 1093 heartbeater.cc:347] Connected to a master server at 127.0.93.190:42771
02:37:49.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.602682 1093 heartbeater.cc:464] Registering TS with master...
02:37:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.603152 1091 heartbeater.cc:347] Connected to a master server at 127.0.93.188:42181
02:37:49.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.603467 1091 heartbeater.cc:464] Registering TS with master...
02:37:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.604713 1091 heartbeater.cc:511] Master 127.0.93.188:42181 requested a full tablet report, sending...
02:37:49.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.604873 1093 heartbeater.cc:511] Master 127.0.93.190:42771 requested a full tablet report, sending...
02:37:49.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.606063 1092 heartbeater.cc:347] Connected to a master server at 127.0.93.189:34179
02:37:49.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.606374 1092 heartbeater.cc:464] Registering TS with master...
02:37:49.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.607501 573 ts_manager.cc:194] Registered new tserver with Master: bbac8d84b06d4f9dbf2a9a1cff7bda2e (127.0.93.131:35463)
02:37:49.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.607514 1092 heartbeater.cc:511] Master 127.0.93.189:34179 requested a full tablet report, sending...
02:37:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.609319 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:45703
02:37:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.609534 419 ts_manager.cc:194] Registered new tserver with Master: bbac8d84b06d4f9dbf2a9a1cff7bda2e (127.0.93.131:35463)
02:37:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.611034 494 ts_manager.cc:194] Registered new tserver with Master: bbac8d84b06d4f9dbf2a9a1cff7bda2e (127.0.93.131:35463)
02:37:49.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:49.618079 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
02:37:49.628 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:37:49.629 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:37:49.640 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 42771
}
bound_http_address {
host: "127.0.93.190"
port: 46069
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 34179
}
bound_http_address {
host: "127.0.93.189"
port: 38051
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 42181
}
bound_http_address {
host: "127.0.93.188"
port: 35371
}
}
}
02:37:49.644 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:37:49.646 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 39783
}
bound_http_address {
host: "127.0.93.129"
port: 36701
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 37349
}
bound_http_address {
host: "127.0.93.130"
port: 42935
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 35463
}
bound_http_address {
host: "127.0.93.131"
port: 36979
}
}
}
02:37:49.646 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:37:49.837 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:37:49.940 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskvhYo2y/test-tmp
02:37:49.941 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:37:49.941 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskvhYo2y/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:37:49.945 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:37:50.275 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:50.275 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:37:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:50.278893 1102 mini_oidc.cc:152] Starting JWKS server
02:37:50.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:50.282577 1102 webserver.cc:460] Webserver started at http://127.0.0.1:33917/ using document root <none> and password file <none>
02:37:50.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:50.283077 1102 mini_oidc.cc:189] Starting OIDC Discovery server
02:37:50.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:50.285238 1102 webserver.cc:460] Webserver started at http://127.0.0.1:37857/ using document root <none> and password file <none>
02:37:50.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:37:50.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/principal' for realm 'KRBTEST.COM',
02:37:50.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:37:50.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:50 dist-test-slave-chs2 krb5kdc[1114](info): setting up network...
02:37:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:37:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:50 dist-test-slave-chs2 krb5kdc[1114](info): set up 2 sockets
02:37:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:50 dist-test-slave-chs2 krb5kdc[1114](info): commencing operation
02:37:50.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:37:50.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:50.613304 1091 heartbeater.cc:503] Master 127.0.93.188:42181 was elected leader, sending a full tablet report...
02:37:51.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.394991 1102 mini_kdc.cc:121] Time spent starting KDC: real 1.084s user 0.008s sys 0.012s
02:37:51.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:37:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:37:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:37:51.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:37:51.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:37:51.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:37:51.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:37:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:37:51.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:37:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:37:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/test-user.keytab.
02:37:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/test-user.keytab.
02:37:51.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:51 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649471, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:37:51.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:37:51.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:37:51Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
02:37:51.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:37:51Z Disabled control of system clock
02:37:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.190@KRBTEST.COM; defaulting to no policy
02:37:51.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:37:51.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.190@KRBTEST.COM" created.
02:37:51.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:37:51.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.190.keytab.
02:37:51.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.190.keytab.
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:51.587065 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:51.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/logs
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/info.pb
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:51.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:33097
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.190.keytab
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:51.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:37:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.894474 1128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.894985 1128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.895529 1128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.926859 1128 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:37:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.927127 1128 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.927443 1128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.927637 1128 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.927830 1128 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:51.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.928007 1128 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:51.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.928282 1128 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33917/jwks.json
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:51.965593 1128 master_runner.cc:386] Master server non-default flags:
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.190.keytab
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:33097
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/info.pb
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
02:37:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/logs
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:51.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:51.966727 1128 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:51.967769 1128 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:51.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.986676 1135 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.986965 1137 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:51.988232 1134 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.263908 1136 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:37:53.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.264061 1128 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:37:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.268970 1128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:53.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.271431 1128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:53.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.272850 1128 hybrid_clock.cc:648] HybridClock initialized: now 1790649473272773 us; error 75 us; skew 500 ppm
02:37:53.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.273810 1128 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:53.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:53 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649473, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:37:53.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.280901 1128 init.cc:377] Logged in from keytab as kudu/127.1.19.190@KRBTEST.COM (short username kudu)
02:37:53.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.286521 1128 webserver.cc:460] Webserver started at http://127.1.19.190:34573/ using document root <none> and password file <none>
02:37:53.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.287600 1128 fs_manager.cc:359] Metadata directory not provided
02:37:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.287834 1128 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:53.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.288372 1128 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.293160 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/instance:
02:37:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88e004ec3c854f3aab0f7cab89a3eb2f"
02:37:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:53 on dist-test-slave-chs2"
02:37:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.294327 1128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal/instance:
02:37:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88e004ec3c854f3aab0f7cab89a3eb2f"
02:37:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:53 on dist-test-slave-chs2"
02:37:53.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.301637 1128 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.007s
02:37:53.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.307282 1146 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:53.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.308342 1128 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
02:37:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.308715 1128 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88e004ec3c854f3aab0f7cab89a3eb2f"
02:37:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:53 on dist-test-slave-chs2"
02:37:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.309075 1128 fs_report.cc:383] FS layout report
02:37:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:53.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/data
02:37:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:53.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:53.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.344357 1128 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:53.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.350375 1128 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:53.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.350999 1128 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.426893 1128 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.190:33097
02:37:53.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.427011 1198 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.190:33097 every 8 connection(s)
02:37:53.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.429764 1128 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/info.pb
02:37:53.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.436506 1199 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:37:53.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.437974 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 1128
02:37:53.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.438345 1102 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal/instance
02:37:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.447636 1199 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } has no permanent_uuid. Determining permanent_uuid...
02:37:53.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.189@KRBTEST.COM; defaulting to no policy
02:37:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:37:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.189@KRBTEST.COM" created.
02:37:53.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:53 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649473, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
02:37:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:37:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.189.keytab.
02:37:53.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.189.keytab.
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.485877 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/logs
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/info.pb
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:53.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:39209
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:53.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.189.keytab
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:37:53.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.494472 1199 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } has no permanent_uuid. Determining permanent_uuid...
02:37:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.497301 1147 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:53.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.501899 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:39209: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:53.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.551379 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } attempt: 1
02:37:53.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.554998 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:39209: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:53.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.623951 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } attempt: 2
02:37:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.629088 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:39209: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:53.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.720531 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } attempt: 3
02:37:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.724637 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:39209: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:53.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.805898 1205 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:53.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.806445 1205 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:53.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.807194 1205 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.839329 1205 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:37:53.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.839581 1205 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.839846 1205 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.840113 1205 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.840323 1205 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.840509 1205 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.840664 1205 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33917/jwks.json
02:37:53.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.867993 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } attempt: 4
02:37:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.872316 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:39209: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:53.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.880455 1205 master_runner.cc:386] Master server non-default flags:
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:53.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.189.keytab
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:39209
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/info.pb
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
02:37:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/logs
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.882032 1205 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.883385 1205 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:53.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.902954 1213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:54.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:54.171720 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } attempt: 5
02:37:54.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:54.176888 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:39209: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:54.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:54.724303 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } attempt: 6
02:37:54.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:54.731194 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.189:39209: Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:37:55.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.902984 1212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:53.905371 1205 server_base.cc:1034] running on GCE node
02:37:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:53.904477 1215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.170466 1205 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:55.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.172915 1205 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:55.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.174256 1205 hybrid_clock.cc:648] HybridClock initialized: now 1790649475174227 us; error 47 us; skew 500 ppm
02:37:55.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.175053 1205 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:55.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:55 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649475, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:37:55.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.181699 1205 init.cc:377] Logged in from keytab as kudu/127.1.19.189@KRBTEST.COM (short username kudu)
02:37:55.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.187695 1205 webserver.cc:460] Webserver started at http://127.1.19.189:40435/ using document root <none> and password file <none>
02:37:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.188824 1205 fs_manager.cc:359] Metadata directory not provided
02:37:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.189075 1205 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:55.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.189579 1205 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.196228 1205 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/instance:
02:37:55.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b19b9c857324389aeae9109d8eec773"
02:37:55.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:55 on dist-test-slave-chs2"
02:37:55.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.197574 1205 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal/instance:
02:37:55.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b19b9c857324389aeae9109d8eec773"
02:37:55.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:55 on dist-test-slave-chs2"
02:37:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.207063 1205 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
02:37:55.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.214212 1225 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:55.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.215384 1205 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.215760 1205 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b19b9c857324389aeae9109d8eec773"
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:55 on dist-test-slave-chs2"
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.216208 1205 fs_report.cc:383] FS layout report
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/data
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:55.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:55.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.254773 1205 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.260418 1205 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:55.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.260813 1205 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:55.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.338065 1205 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.189:39209
02:37:55.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.338099 1276 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.189:39209 every 8 connection(s)
02:37:55.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.340715 1205 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/info.pb
02:37:55.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.346710 1277 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:37:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.349941 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 1205
02:37:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.350433 1102 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal/instance
02:37:55.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.362144 1277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } has no permanent_uuid. Determining permanent_uuid...
02:37:55.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.188@KRBTEST.COM; defaulting to no policy
02:37:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:37:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.188@KRBTEST.COM" created.
02:37:55.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:55 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649475, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
02:37:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:37:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.188.keytab.
02:37:55.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.188.keytab.
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.397518 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/logs
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/info.pb
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:40397
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.188.keytab
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:55.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:37:55.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.399338 1277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } has no permanent_uuid. Determining permanent_uuid...
02:37:55.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:55 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649475, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
02:37:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.431365 1277 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } has no permanent_uuid. Determining permanent_uuid...
02:37:55.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.434330 1228 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.438894 1277 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.488385 1277 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 1
02:37:55.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.492004 1277 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.560423 1277 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 2
02:37:55.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.564539 1277 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.655977 1277 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 3
02:37:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.659546 1277 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.726897 1284 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:55.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.727476 1284 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:55.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.728068 1284 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.763311 1284 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:37:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.763533 1284 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:55.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.763782 1284 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.763967 1284 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.764232 1284 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.764417 1284 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:55.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.764549 1284 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33917/jwks.json
02:37:55.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.791833 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } attempt: 7
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.802228 1284 master_runner.cc:386] Master server non-default flags:
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:37:55.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.188.keytab
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:40397
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/info.pb
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/logs
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:55.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.802932 1277 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 4
02:37:55.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.803488 1284 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.804818 1284 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:55.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:55 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649473, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
02:37:55.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.806839 1277 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.820158 1199 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } has no permanent_uuid. Determining permanent_uuid...
02:37:55.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.826946 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.829432 1295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:55.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.885371 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 1
02:37:55.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.889328 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:55.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:55.970705 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 2
02:37:55.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.977239 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:56.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:56.062794 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 3
02:37:56.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:56.067929 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:56.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:56.106191 1277 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 5
02:37:56.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:56.110371 1277 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:56.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:56.208506 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 4
02:37:56.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:56.212145 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:56.495545 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 5
02:37:56.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:56.499380 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:56.657732 1277 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 6
02:37:56.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:56.666539 1277 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:57.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.051980 1199 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } attempt: 6
02:37:57.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.060032 1199 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.188:40397: Network error: Client connection negotiation failed: client connection to 127.1.19.188:40397: connect: Connection refused (error 111)
02:37:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.829983 1292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:55.829437 1293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.089095 1294 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
02:37:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.089213 1284 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:37:57.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.094080 1284 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.097000 1284 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.098459 1284 hybrid_clock.cc:648] HybridClock initialized: now 1790649477098398 us; error 42 us; skew 500 ppm
02:37:57.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.099298 1284 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649477, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:37:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.105053 1284 init.cc:377] Logged in from keytab as kudu/127.1.19.188@KRBTEST.COM (short username kudu)
02:37:57.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.110363 1284 webserver.cc:460] Webserver started at http://127.1.19.188:35981/ using document root <none> and password file <none>
02:37:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.111171 1284 fs_manager.cc:359] Metadata directory not provided
02:37:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.111353 1284 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.111719 1284 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
02:37:57.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.116328 1284 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/instance:
02:37:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "707eb1ea0c6f4e119caf7539c7331d7d"
02:37:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:57 on dist-test-slave-chs2"
02:37:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.117327 1284 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal/instance:
02:37:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "707eb1ea0c6f4e119caf7539c7331d7d"
02:37:57.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:57 on dist-test-slave-chs2"
02:37:57.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.124568 1284 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
02:37:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.130026 1311 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:57.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.131251 1284 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:37:57.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.131585 1284 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:37:57.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "707eb1ea0c6f4e119caf7539c7331d7d"
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:57 on dist-test-slave-chs2"
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.131919 1284 fs_report.cc:383] FS layout report
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/data
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:57.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:57.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:57.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:57.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:57.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.209731 1284 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:57.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.216185 1284 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:57.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.216591 1284 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.294390 1284 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.188:40397
02:37:57.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.294463 1362 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.188:40397 every 8 connection(s)
02:37:57.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.297123 1284 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/info.pb
02:37:57.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.300874 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 1284
02:37:57.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.301364 1102 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal/instance
02:37:57.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.304280 1363 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:37:57.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.319151 1363 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } has no permanent_uuid. Determining permanent_uuid...
02:37:57.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649471, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.188@KRBTEST.COM
02:37:57.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649477, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
02:37:57.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.356137 1363 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } has no permanent_uuid. Determining permanent_uuid...
02:37:57.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649477, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
02:37:57.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649471, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.189@KRBTEST.COM
02:37:57.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.381719 1363 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } has no permanent_uuid. Determining permanent_uuid...
02:37:57.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649477, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for kudu/127.1.19.188@KRBTEST.COM
02:37:57.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649471, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.190@KRBTEST.COM
02:37:57.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.416069 1363 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Bootstrap starting.
02:37:57.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.423254 1363 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Neither blocks nor log segments found. Creating new log.
02:37:57.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.426064 1363 log.cc:824] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Log is configured to *not* fsync() on all Append() calls
02:37:57.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.427819 1102 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
02:37:57.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.431797 1363 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: No bootstrap required, opened a new log
02:37:57.439 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:57.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:37:57.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:37:57.457 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:37:57.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.460260 1363 raft_consensus.cc:348] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:37:57.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.461140 1363 raft_consensus.cc:374] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:37:57.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.461865 1363 raft_consensus.cc:729] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 707eb1ea0c6f4e119caf7539c7331d7d, State: Initialized, Role: FOLLOWER
02:37:57.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.190"
port: 33097
}
bound_http_address {
host: "127.1.19.190"
port: 34573
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.189"
port: 39209
}
bound_http_address {
host: "127.1.19.189"
port: 40435
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.188"
port: 40397
}
bound_http_address {
host: "127.1.19.188"
port: 35981
}
}
}
02:37:57.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.462811 1363 consensus_queue.cc:260] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:37:57.463 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:37:57.464 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:57.464 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:37:57.466 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:37:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.465715 1372 sys_catalog.cc:455] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } } }
02:37:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.466996 1372 sys_catalog.cc:458] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [sys.catalog]: This master's current role is: FOLLOWER
02:37:57.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.468092 1363 sys_catalog.cc:564] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [sys.catalog]: configured and running, proceeding with master startup.
02:37:57.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.472105 1317 tablet.cc:2369] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:37:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649477, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:37:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:37:57.511 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:57.514 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc
02:37:57.524 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:37:57.526 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc
02:37:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.531066 1385 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:37:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.531383 1385 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:37:57.534 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:37:57.540 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:37:57.544 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:37:57.544 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:37:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.549345 1372 raft_consensus.cc:482] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:37:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.549901 1372 raft_consensus.cc:504] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:37:57.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.552767 1372 leader_election.cc:288] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097), 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209)
02:37:57.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.526890 1284 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: OK, instance_id { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" instance_seqno: 1790649473406894 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:37:57.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.567696 1284 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:37:57.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:57 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649477, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:37:57.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:37:57.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:57.602 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc
02:37:57.604 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:37:57.604 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.190:33097
02:37:57.606 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:37:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.607203 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 1128
02:37:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.635986 1312 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.1.19.190:33097 (error 108)
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.642647 1368 negotiation.cc:337] Failed RPC negotiation. Trace:
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 02:37:57.639883 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.1.19.190:33097 (local address 127.0.0.1:49096)
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 02:37:57.640380 (+ 497us) negotiation.cc:107] Waiting for socket to connect
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 02:37:57.640422 (+ 42us) client_negotiation.cc:174] Beginning negotiation
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 02:37:57.640635 (+ 213us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0929 02:37:57.641373 (+ 738us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":74}
02:37:57.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:57.643101 1312 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097): Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
02:37:57.645 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:57.645 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.188:40397
02:37:57.645 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:37:57.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.646210 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 1284
02:37:57.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:57.677 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.189:39209
02:37:57.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:37:57.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.677909 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 1205
02:37:57.699 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:57.699 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.190:33097
02:37:57.701 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:37:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:57.703191 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data
02:37:57.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:57.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:57.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/logs
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/info.pb
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:33097
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34573
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.190.keytab
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:57.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:57.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:37:58.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.006301 1390 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:58.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.006984 1390 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:58.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.007735 1390 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:58.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.038439 1390 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:37:58.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.038724 1390 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:58.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.039161 1390 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:58.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.039592 1390 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.039908 1390 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.040241 1390 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.040494 1390 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33917/jwks.json
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:58.077171 1390 master_runner.cc:386] Master server non-default flags:
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:58.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.190.keytab
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.190
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.190:33097
02:37:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/info.pb
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.190
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34573
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/logs
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:58.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:58.078735 1390 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:58.080119 1390 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:58.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.097255 1398 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.097262 1397 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:59.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:58.098912 1390 server_base.cc:1034] running on GCE node
02:37:59.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:58.097353 1400 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:37:59.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.321280 1390 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:37:59.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.324330 1390 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:37:59.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.325743 1390 hybrid_clock.cc:648] HybridClock initialized: now 1790649479325710 us; error 43 us; skew 500 ppm
02:37:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.326514 1390 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:37:59.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:37:59 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649479, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:37:59.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.339414 1390 init.cc:377] Logged in from keytab as kudu/127.1.19.190@KRBTEST.COM (short username kudu)
02:37:59.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.344671 1390 webserver.cc:460] Webserver started at http://127.1.19.190:34573/ using document root <none> and password file <none>
02:37:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.345564 1390 fs_manager.cc:359] Metadata directory not provided
02:37:59.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.345762 1390 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:37:59.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.353351 1390 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
02:37:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.358021 1409 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:37:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.359100 1390 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:37:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.359383 1390 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88e004ec3c854f3aab0f7cab89a3eb2f"
02:37:59.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:53 on dist-test-slave-chs2"
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.361560 1390 fs_report.cc:383] FS layout report
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/data
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:37:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:37:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:37:59.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.413784 1390 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:37:59.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.419550 1390 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:37:59.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.420233 1390 kserver.cc:163] Server-wide thread pool size limit: 3276
02:37:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.489912 1390 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.190:33097
02:37:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.490051 1460 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.190:33097 every 8 connection(s)
02:37:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.493630 1390 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/data/info.pb
02:37:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.502494 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 1390
02:37:59.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:37:59.503 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.188:40397
02:37:59.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.505053 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/logs
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/info.pb
02:37:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:40397
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35981
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.188.keytab
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:37:59.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.507033 1461 sys_catalog.cc:263] Verifying existing consensus state
02:37:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 02:37:59.508126 1461 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:37:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.508703 1390 master.cc:537] Master@127.1.19.190:33097 shutting down...
02:37:59.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.572713 1390 master.cc:552] Master@127.1.19.190:33097 shutdown complete.
02:37:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:37:59.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.856824 1463 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:37:59.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.857409 1463 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:37:59.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.858039 1463 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:37:59.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.898011 1463 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:37:59.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.898222 1463 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:37:59.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.898476 1463 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:37:59.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.898797 1463 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:37:59.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.899112 1463 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:37:59.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.899441 1463 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:37:59.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.899706 1463 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33917/jwks.json
02:37:59.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.937294 1463 master_runner.cc:386] Master server non-default flags:
02:37:59.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.188.keytab
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.188
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.188:40397
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/info.pb
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.188
02:37:59.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35981
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/logs
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.938365 1463 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:37:59.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.939343 1463 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:37:59.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.957149 1469 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:38:01.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.957696 1472 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:38:01.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:37:59.960091 1463 server_base.cc:1034] running on GCE node
02:38:01.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:37:59.958350 1470 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:38:01.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.181666 1463 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:38:01.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.191741 1463 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:38:01.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.193195 1463 hybrid_clock.cc:648] HybridClock initialized: now 1790649481193042 us; error 160 us; skew 500 ppm
02:38:01.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.194226 1463 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:38:01.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:38:01 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649481, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:38:01.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.201081 1463 init.cc:377] Logged in from keytab as kudu/127.1.19.188@KRBTEST.COM (short username kudu)
02:38:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.206605 1463 webserver.cc:460] Webserver started at http://127.1.19.188:35981/ using document root <none> and password file <none>
02:38:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.207546 1463 fs_manager.cc:359] Metadata directory not provided
02:38:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.207804 1463 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:38:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.215399 1463 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.005s
02:38:01.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.219544 1482 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:38:01.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.220541 1463 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.220903 1463 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "707eb1ea0c6f4e119caf7539c7331d7d"
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:57 on dist-test-slave-chs2"
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.223134 1463 fs_report.cc:383] FS layout report
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/wal
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/data
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:38:01.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:38:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:38:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:38:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:38:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:38:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:38:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:38:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:38:01.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.259338 1463 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:38:01.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.265092 1463 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:38:01.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.265611 1463 kserver.cc:163] Server-wide thread pool size limit: 3276
02:38:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.336438 1463 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.188:40397
02:38:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.336624 1533 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.188:40397 every 8 connection(s)
02:38:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.339111 1463 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-2/data/info.pb
02:38:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.341670 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 1463
02:38:01.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:38:01.343 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.189:39209
02:38:01.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:38:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.345256 1102 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:38:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu
02:38:01.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/logs
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/info.pb
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:38:01.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:39209
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40435
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.189.keytab
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:38:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:38:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.351794 1534 sys_catalog.cc:263] Verifying existing consensus state
02:38:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.357591 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Bootstrap starting.
02:38:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.389977 1534 log.cc:824] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Log is configured to *not* fsync() on all Append() calls
02:38:01.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.394409 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:38:01.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.395148 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Bootstrap complete.
02:38:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.418757 1534 raft_consensus.cc:348] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.421087 1534 raft_consensus.cc:374] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:38:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.421415 1534 raft_consensus.cc:729] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 707eb1ea0c6f4e119caf7539c7331d7d, State: Initialized, Role: FOLLOWER
02:38:01.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.422272 1534 consensus_queue.cc:260] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:01.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.425153 1538 sys_catalog.cc:455] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } } }
02:38:01.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.425983 1538 sys_catalog.cc:458] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [sys.catalog]: This master's current role is: FOLLOWER
02:38:01.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.426743 1534 sys_catalog.cc:564] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [sys.catalog]: configured and running, proceeding with master startup.
02:38:01.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.458315 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.458906 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.461225 1463 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111),
02:38:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.483103 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:38:01.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.501032 1538 raft_consensus.cc:482] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:38:01.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.501557 1538 raft_consensus.cc:504] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.504575 1538 leader_election.cc:288] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097), 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209)
02:38:01.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.507752 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111)
02:38:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.513729 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097): Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111)
02:38:01.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.517328 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209): Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:38:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.518188 1483 leader_election.cc:302] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 707eb1ea0c6f4e119caf7539c7331d7d; no voters: 2b19b9c857324389aeae9109d8eec773, 88e004ec3c854f3aab0f7cab89a3eb2f
02:38:01.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.519593 1538 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:38:01.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.520681 1488 tablet.cc:2369] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:38:01.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.687510 1536 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
02:38:01.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.688113 1536 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:38:01.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.688722 1536 flags.cc:425] Enabled unsafe flag: --never_fsync=true
02:38:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.719741 1536 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
02:38:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.719978 1536 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
02:38:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.720323 1536 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
02:38:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.720535 1536 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
02:38:01.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.720723 1536 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
02:38:01.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.720909 1536 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
02:38:01.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.721050 1536 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33917/jwks.json
02:38:01.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.759521 1536 master_runner.cc:386] Master server non-default flags:
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.148:42993
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:38:01.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.190:33097,127.1.19.189:39209,127.1.19.188:40397
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/krb5kdc/kudu_127.1.19.189.keytab
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.189
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.189:39209
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33917/jwks.json
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/info.pb
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.189
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40435
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/logs
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:38:01.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 01:48:57 UTC on ccycloud.quasar-czwlgj.root.comops.site
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679061
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.760708 1536 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:38:01.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.761737 1536 file_cache.cc:493] Constructed file cache file cache with capacity 419430
02:38:01.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.783749 1560 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:38:02.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:02.465286 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:02.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:02.465799 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.784830 1559 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:01.786031 1536 server_base.cc:1034] running on GCE node
02:38:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:01.783871 1562 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:38:03.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.063560 1536 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:38:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.066606 1536 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:38:03.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.068146 1536 hybrid_clock.cc:648] HybridClock initialized: now 1790649483068085 us; error 85 us; skew 500 ppm
02:38:03.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.068975 1536 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:38:03.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:38:03 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649483, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:38:03.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.074951 1536 init.cc:377] Logged in from keytab as kudu/127.1.19.189@KRBTEST.COM (short username kudu)
02:38:03.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.081516 1536 webserver.cc:460] Webserver started at http://127.1.19.189:40435/ using document root <none> and password file <none>
02:38:03.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.082504 1536 fs_manager.cc:359] Metadata directory not provided
02:38:03.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.082751 1536 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:38:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.091544 1536 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.000s
02:38:03.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.096355 1570 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:38:03.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.097569 1536 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
02:38:03.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.097911 1536 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data,/tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:38:03.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b19b9c857324389aeae9109d8eec773"
02:38:03.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 02:37:55 on dist-test-slave-chs2"
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.100386 1536 fs_report.cc:383] FS layout report
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/data
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:38:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:38:03.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.168866 1536 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
02:38:03.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.174430 1536 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:38:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.174821 1536 kserver.cc:163] Server-wide thread pool size limit: 3276
02:38:03.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.197540 1578 raft_consensus.cc:482] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:38:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.198423 1578 raft_consensus.cc:504] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.207106 1578 leader_election.cc:288] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097), 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209)
02:38:03.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:03.230768 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097): Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111)
02:38:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:03.233485 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209): Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:38:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.234086 1483 leader_election.cc:302] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 707eb1ea0c6f4e119caf7539c7331d7d; no voters: 2b19b9c857324389aeae9109d8eec773, 88e004ec3c854f3aab0f7cab89a3eb2f
02:38:03.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.235286 1578 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:38:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.300196 1536 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.189:39209
02:38:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.300625 1626 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.189:39209 every 8 connection(s)
02:38:03.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.302917 1536 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/data/info.pb
02:38:03.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.307595 1102 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu as pid 1536
02:38:03.308 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:38:03.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.313863 1627 sys_catalog.cc:263] Verifying existing consensus state
02:38:03.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 02:38:03.314670 1627 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:38:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.315197 1536 master.cc:537] Master@127.1.19.189:39209 shutting down...
02:38:03.370 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:03.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:03.394629 1536 master.cc:552] Master@127.1.19.189:39209 shutdown complete.
02:38:03.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskvhYo2y/test-tmp/mini-kudu-cluster1202742776192046320/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
02:38:03.438 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.188:40397(127.1.19.188:40397)] connecting to peer
02:38:03.441 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:03.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:03.469975 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:03.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:03.470324 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:03.499 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.19.188:40397(127.1.19.188:40397)] Successfully connected to peer
02:38:03.502 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.506 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.506 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.507 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.582 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:38:03.591 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.188
02:38:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:38:03 dist-test-slave-chs2 krb5kdc[1114](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649477, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.188@KRBTEST.COM
02:38:03.805 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x7c5837a5, L:/127.0.0.1:58186 - R:/127.1.19.188:40397] using SASL/GSSAPI
02:38:03.819 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:03.833 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:03.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.838 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:03.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.841 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.841 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.843 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:03.852 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:03.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:03.855 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.856 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.856 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.857 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.857 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.860 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:03.872 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:03.874 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:03.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.880 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.882 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.882 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.882 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.883 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:03.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:03.915 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:03.917 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.917 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.917 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.917 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.918 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.920 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:03.933 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:03.934 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:03.936 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.937 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.937 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.937 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.937 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.938 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.939 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:03.973 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:03.975 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:03.977 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.977 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.978 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:03.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:03.981 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:04.012 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:04.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:04.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.016 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.017 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.019 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:04.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:04.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:04.077 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.077 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.079 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.080 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.081 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.082 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:04.372 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:04.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:04.376 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.376 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.376 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.376 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.377 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.377 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.378 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:04.473191 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:04.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:04.473474 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:04.872 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:04.876 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.877 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.877 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:04.877 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:04.880 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.880 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.880 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:04.880 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:05.063961 1633 raft_consensus.cc:482] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:38:05.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:05.064309 1633 raft_consensus.cc:504] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:05.065760 1633 leader_election.cc:288] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097), 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209)
02:38:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:05.070479 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097): Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111)
02:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:05.070896 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209): Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:05.071122 1483 leader_election.cc:302] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 707eb1ea0c6f4e119caf7539c7331d7d; no voters: 2b19b9c857324389aeae9109d8eec773, 88e004ec3c854f3aab0f7cab89a3eb2f
02:38:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:05.071620 1633 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:38:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:05.476306 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:05.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:05.476591 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:06.479672 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:06.479946 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:06.486016 1463 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111),
02:38:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:06.486464 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:38:06.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:06.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:06.896 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:06.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:06.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:06.897 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:06.897 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:06.897 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:06.898 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:07.478545 1641 raft_consensus.cc:482] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:38:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:07.478998 1641 raft_consensus.cc:504] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:07.481099 1641 leader_election.cc:288] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097), 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209)
02:38:07.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:07.483568 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:07.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:07.484000 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:07.485086 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111) [suppressed 5 similar messages]
02:38:07.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:07.488587 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097): Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111)
02:38:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:07.489187 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209): Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:38:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:07.489548 1483 leader_election.cc:302] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 707eb1ea0c6f4e119caf7539c7331d7d; no voters: 2b19b9c857324389aeae9109d8eec773, 88e004ec3c854f3aab0f7cab89a3eb2f
02:38:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:07.490252 1641 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:38:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:08.487443 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:08.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:08.487740 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:09.013 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:09.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:09.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:09.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:09.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:09.017 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:09.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:09.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:09.018 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:09.490500 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:09.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:09.490739 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:09.572 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:09.574 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:09.575 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:09.575 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:09.576 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:09.576 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:09.576 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:09.576 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:09.577 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:10.494182 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:10.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:10.494506 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:38:11 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649491, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:38:11.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:11.205031 1478 init.cc:295] Successfully reacquired a new kerberos TGT
02:38:11.392 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:11.395 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:11.395 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:11.396 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:11.396 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:11.397 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:11.397 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:11.397 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:11.398 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:11.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:11.489934 1463 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111),
02:38:11.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:11.490350 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:38:11.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:11.497406 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:11.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:11.497675 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:12.500511 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:12.500811 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:12.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:12.952560 1087 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6620 after lost signal to thread 960
02:38:12.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:12.953428 1087 debug-util.cc:398] Leaking SignalData structure 0x7b08000af0c0 after lost signal to thread 1090
02:38:13.253 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:13.255 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:13.258 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:13.258 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:13.258 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:13.258 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:13.259 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:13.259 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:13.260 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:13.277693 1653 raft_consensus.cc:482] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:38:13.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:13.278234 1653 raft_consensus.cc:504] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:13.280124 1653 leader_election.cc:288] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097), 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209)
02:38:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:13.282706 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:38:13.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:13.286307 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097): Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111)
02:38:13.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:13.287102 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209): Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:38:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:13.287379 1483 leader_election.cc:302] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 707eb1ea0c6f4e119caf7539c7331d7d; no voters: 2b19b9c857324389aeae9109d8eec773, 88e004ec3c854f3aab0f7cab89a3eb2f
02:38:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:13.297142 1653 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:38:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:13.504927 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:13.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:13.505249 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:14.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:14.509523 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:14.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:14.509871 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:15.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:15.512938 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:15.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:15.513487 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:15.553 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:15.554 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:15.556 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:15.556 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:15.556 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:15.556 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:15.556 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:15.557 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:15.558 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:16.053 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:16.054 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:16.056 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:16.056 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:16.057 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:16.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:16.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:16.058 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:16.061 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:16.492745 1463 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111),
02:38:16.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:16.493147 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:38:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:16.516321 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:16.516590 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:17.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:17.520532 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:17.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:17.520794 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:18.523432 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:18.523713 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:18.753 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:18.754 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:18.756 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:18.756 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:18.756 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:18.758 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:18.758 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:18.758 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:18.759 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:18.784044 605 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1f20 after lost signal to thread 536
02:38:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:18.784571 605 debug-util.cc:398] Leaking SignalData structure 0x7b0800090ca0 after lost signal to thread 608
02:38:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:19.427970 804 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbb20 after lost signal to thread 677
02:38:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:19.428828 804 debug-util.cc:398] Leaking SignalData structure 0x7b08000af0c0 after lost signal to thread 807
02:38:19.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:19.528105 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:19.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:19.528452 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:20.531891 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:20.532274 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:21.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:21.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:21.076 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:21.076 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:21.076 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:21.077 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:21.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:21.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:21.079 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:38:21 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649501, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:38:21.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:21.207499 1478 init.cc:295] Successfully reacquired a new kerberos TGT
02:38:21.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:21.425289 605 debug-util.cc:398] Leaking SignalData structure 0x7b08000767c0 after lost signal to thread 536
02:38:21.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:21.426002 605 debug-util.cc:398] Leaking SignalData structure 0x7b080007f440 after lost signal to thread 608
02:38:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:21.496312 1463 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111),
02:38:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:21.496694 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:38:21.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:21.534982 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:21.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:21.535241 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:22.072 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:22.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:22.075 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:22.075 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:22.075 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:22.076 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:22.077 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:22.077 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:22.077 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:22.538079 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:22.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:22.538321 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:22.762396 1669 raft_consensus.cc:482] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:38:22.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:22.762827 1669 raft_consensus.cc:504] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88e004ec3c854f3aab0f7cab89a3eb2f" member_type: VOTER last_known_addr { host: "127.1.19.190" port: 33097 } } peers { permanent_uuid: "2b19b9c857324389aeae9109d8eec773" member_type: VOTER last_known_addr { host: "127.1.19.189" port: 39209 } } peers { permanent_uuid: "707eb1ea0c6f4e119caf7539c7331d7d" member_type: VOTER last_known_addr { host: "127.1.19.188" port: 40397 } }
02:38:22.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:22.764204 1669 leader_election.cc:288] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097), 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209)
02:38:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:22.766660 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111) [suppressed 1 similar messages]
02:38:22.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:22.769614 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 88e004ec3c854f3aab0f7cab89a3eb2f (127.1.19.190:33097): Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111)
02:38:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:22.770179 1483 leader_election.cc:334] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2b19b9c857324389aeae9109d8eec773 (127.1.19.189:39209): Network error: Client connection negotiation failed: client connection to 127.1.19.189:39209: connect: Connection refused (error 111)
02:38:22.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:22.770483 1483 leader_election.cc:302] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 707eb1ea0c6f4e119caf7539c7331d7d; no voters: 2b19b9c857324389aeae9109d8eec773, 88e004ec3c854f3aab0f7cab89a3eb2f
02:38:22.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:22.771140 1669 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
02:38:23.312 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:23.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:23.314 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:23.315 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:23.315 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:23.315 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:23.315 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:23.315 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:23.317 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:23.541118 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:23.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:23.541369 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:24.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:24.544730 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:24.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:24.544979 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:25.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:25.547567 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:25.547811 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:26.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:26.499356 1463 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111),
02:38:26.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:26.499811 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:38:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:26.550628 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:26.550932 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:26.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:26.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:26.895 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:26.895 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:26.895 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:26.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:26.895 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:26.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:26.897 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:27.553740 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:27.553998 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:28.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:28.556974 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:28.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:28.557214 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:28.852 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:28.853 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:28.854 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:28.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:28.855 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:28.855 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:28.855 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:28.855 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:28.857 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:29.559990 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:29.560334 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:30.564069 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:30.564393 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 02:38:31 dist-test-slave-chs2 krb5kdc[1114](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790649511, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:38:31.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:31.209941 1478 init.cc:295] Successfully reacquired a new kerberos TGT
02:38:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:31.503103 1463 master_runner.cc:186] Error getting master registration for 127.1.19.190:33097: Network error: Client connection negotiation failed: client connection to 127.1.19.190:33097: connect: Connection refused (error 111),
02:38:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:31.503525 1463 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
02:38:31.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:31.566932 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 707eb1ea0c6f4e119caf7539c7331d7d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:38:31.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 02:38:31.567186 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:38:31.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.190:33097(127.1.19.190:33097)] connecting to peer
02:38:31.814 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.189:39209(127.1.19.189:39209)] connecting to peer
02:38:31.815 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:31.815 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:31.815 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.190:33097(127.1.19.190:33097)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:31.815 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.189:39209(127.1.19.189:39209)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:31.815 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.190:33097: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097
02:38:31.815 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.189:39209: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
02:38:31.817 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209; will retry
02:38:31.822 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28506), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [189ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [498ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [524ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [527ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [540ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [543ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [566ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [760ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1055ms] refreshing cache from master, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1555ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1564ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3575ms] refreshing cache from master, [3576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [3581ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [3582ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [5695ms] refreshing cache from master, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [5697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [5699ms] too many traces: truncated at 100 traces, deferred=Deferred@39290636(state=PENDING, result=null, callback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122), errback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122)))
02:38:31.826 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.190:33097(127.1.19.190:33097)] disconnecting while in state TERMINATED
02:38:31.827 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.189:39209(127.1.19.189:39209)] disconnecting while in state TERMINATED
02:38:31.827 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.188:40397(127.1.19.188:40397)] disconnecting while in state READY
02:38:31.832 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:38:31.833 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.19.188:40397(127.1.19.188:40397)] handling channelInactive
02:38:31.834 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.188:40397(127.1.19.188:40397)] cleaning up while in state READY due to: connection closed
02:38:31.835 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:38:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:31.840638 1102 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 1463
02:38:33.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:38:32Z chronyd exiting
02:38:34.050 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:38:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:34.051637 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 655
02:38:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:34.078470 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 818
02:38:34.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:34.104315 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 956
02:38:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:34.134379 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 386
02:38:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:34.163453 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 456
02:38:34.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 02:38:34.192902 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskvhYo2y/build/tsan/bin/kudu with pid 531
02:38:35.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T02:38:35Z chronyd exiting
02:38:36.374 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28506), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [189ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [498ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [524ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [527ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [540ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [543ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [566ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [760ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1055ms] refreshing cache from master, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1555ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1564ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3575ms] refreshing cache from master, [3576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [3581ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [3582ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [5695ms] refreshing cache from master, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [5697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [5699ms] too many traces: truncated at 100 traces, deferred=Deferred@39290636(state=PENDING, result=null, callback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122), errback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.189:39209
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28506), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [189ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [498ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [524ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [527ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [540ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [543ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [566ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [760ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1055ms] refreshing cache from master, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1555ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1564ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3575ms] refreshing cache from master, [3576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [3581ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [3582ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [5695ms] refreshing cache from master, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [5697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [5699ms] too many traces: truncated at 100 traces, deferred=Deferred@39290636(state=PENDING, result=null, callback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122), errback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.189:39209
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
02:38:37.182 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:38:37.182 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28506), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [189ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [498ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [524ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [527ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [540ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [543ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [566ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [760ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1055ms] refreshing cache from master, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1555ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1564ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3575ms] refreshing cache from master, [3576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [3581ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [3582ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [5695ms] refreshing cache from master, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [5697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [5699ms] too many traces: truncated at 100 traces, deferred=Deferred@39290636(state=PENDING, result=null, callback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122), errback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.189:39209
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28506), Traces: [0ms] refreshing cache from master, [43ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [189ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [498ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [515ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [521ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [524ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [526ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [527ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [535ms] refreshing cache from master, [535ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [536ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [539ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [540ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [543ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [544ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [555ms] refreshing cache from master, [555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [564ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [566ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [601ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [603ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [620ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [622ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [702ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [760ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1055ms] refreshing cache from master, [1055ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1059ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1061ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1062ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1555ms] refreshing cache from master, [1555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [1556ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [1560ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [1560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [1562ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [1563ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [1564ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3575ms] refreshing cache from master, [3576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [3577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.189:39209: Network error: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209, [3580ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.190:33097: Network error: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [3581ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.188:40397: OK, [3582ms] delaying RPC due to: Service unavailable: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097, [5695ms] refreshing cache from master, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.190:33097, [5696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.188:40397, [5697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.189:39209, [5699ms] too many traces: truncated at 100 traces, deferred=Deferred@39290636(state=PENDING, result=null, callback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122), errback=(continuation of Deferred@388459750 after retry RPC after error@805419884) -> (continuation of Deferred@368157458 after retry RPC after error@731920711) -> (continuation of Deferred@130822281 after retry RPC after error@984293831) -> (continuation of Deferred@576388275 after retry RPC after error@144348094) -> (continuation of Deferred@1848947292 after retry RPC after error@233964356) -> (continuation of Deferred@2025380141 after retry RPC after error@618898151) -> (continuation of Deferred@1871594101 after retry RPC after error@1929086179) -> (continuation of Deferred@1286887968 after retry RPC after error@1979264459) -> (continuation of Deferred@1752782059 after retry RPC after error@1183521179) -> (continuation of Deferred@679489072 after retry RPC after error@346802590) -> (continuation of Deferred@920062444 after retry RPC after error@680429932) -> (continuation of Deferred@22467555 after retry RPC after error@1266355566) -> (continuation of Deferred@1164830084 after retry RPC after error@435470764) -> (continuation of Deferred@780845242 after retry RPC after error@1032435832) -> (continuation of Deferred@400698248 after retry RPC after error@681861497) -> (continuation of Deferred@1356069831 after retry RPC after error@543806429) -> (continuation of Deferred@642843187 after retry RPC after error@771035177) -> (continuation of Deferred@1656212371 after retry RPC after error@204348195) -> (continuation of Deferred@574672911 after retry RPC after error@657582134) -> (continuation of Deferred@2072396816 after retry RPC after error@126725952) -> (continuation of Deferred@602523414 after retry RPC after error@196213509) -> (continuation of Deferred@1121880231 after retry RPC after error@675534867) -> (continuation of Deferred@1165982586 after retry RPC after error@1091754243) -> (continuation of Deferred@2000677430 after retry RPC after error@1099573122)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.190:33097,127.1.19.188:40397,127.1.19.189:39209) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.190:33097(127.1.19.190:33097): Connection refused: /127.1.19.190:33097,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.189:39209(127.1.19.189:39209): Connection refused: /127.1.19.189:39209
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.189:39209
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)