05:23:11.275 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:23:11.286 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task55sErp/test-tmp
05:23:11.289 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:23:11.292 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task55sErp/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:23:11.511 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:23:11.696 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:23:11.697 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:23:11.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:11.700692 374 mini_oidc.cc:152] Starting JWKS server
05:23:11.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:11.704603 374 webserver.cc:460] Webserver started at http://127.0.0.1:40627/ using document root <none> and password file <none>
05:23:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:11.705072 374 mini_oidc.cc:189] Starting OIDC Discovery server
05:23:11.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:11.707147 374 webserver.cc:460] Webserver started at http://127.0.0.1:44539/ using document root <none> and password file <none>
05:23:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:23:11Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:23:11.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:23:11Z Disabled control of system clock
05:23:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:11.755177 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/wal
05:23:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data
05:23:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:11.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/logs
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data/info.pb
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:40373
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40627/jwks.json with env {}
05:23:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.053146 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:12.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.053665 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:12.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.054304 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:12.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.084431 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.084687 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.084975 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.085243 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.085542 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:12.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.085768 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40627/jwks.json
05:23:12.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:12.119266 386 master_runner.cc:386] Master server non-default flags:
05:23:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data
05:23:12.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/wal
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:40373
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40627/jwks.json
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data/info.pb
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/logs
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:12.120712 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:12.122625 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.143499 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.143567 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:12.143508 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:12.144492 386 server_base.cc:1034] running on GCE node
05:23:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.364390 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:13.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.367877 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:13.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.369391 386 hybrid_clock.cc:648] HybridClock initialized: now 1782019393369358 us; error 53 us; skew 500 ppm
05:23:13.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.370251 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.378122 386 webserver.cc:460] Webserver started at http://127.0.93.190:34459/ using document root <none> and password file <none>
05:23:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.379130 386 fs_manager.cc:359] Metadata directory not provided
05:23:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.379324 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.379742 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.385365 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data/instance:
05:23:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbdd71e477214e53847b9cf6dbb81c1d"
05:23:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:13 on dist-test-slave-n3h2"
05:23:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.386605 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/wal/instance:
05:23:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbdd71e477214e53847b9cf6dbb81c1d"
05:23:13.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:13 on dist-test-slave-n3h2"
05:23:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.394487 386 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
05:23:13.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.401141 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:13.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.402186 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:23:13.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.402565 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/wal
05:23:13.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fbdd71e477214e53847b9cf6dbb81c1d"
05:23:13.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:13 on dist-test-slave-n3h2"
05:23:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.402993 386 fs_report.cc:383] FS layout report
05:23:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/wal
05:23:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/wal
05:23:13.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data/data
05:23:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:13.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:13.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:13.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:13.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.444150 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:13.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.445909 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:13.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.446475 386 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.530277 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:40373
05:23:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.530364 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:40373 every 8 connection(s)
05:23:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.533169 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/data/info.pb
05:23:13.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.539178 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:23:13.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.542411 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 386
05:23:13.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.542855 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-0/wal/instance
05:23:13.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.547521 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/wal
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/logs
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data/info.pb
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:41975
05:23:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:13.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:13.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40627/jwks.json with env {}
05:23:13.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.551268 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } has no permanent_uuid. Determining permanent_uuid...
05:23:13.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.589859 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } has no permanent_uuid. Determining permanent_uuid...
05:23:13.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.592607 404 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.597541 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41975: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:13.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.647117 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } attempt: 1
05:23:13.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.651322 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41975: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:13.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.719715 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } attempt: 2
05:23:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.724212 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41975: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.815621 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } attempt: 3
05:23:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.819733 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41975: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:13.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.886267 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.886786 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:13.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.887228 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:13.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.920333 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.920692 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.921000 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.921350 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.921720 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:13.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.921978 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40627/jwks.json
05:23:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.958101 456 master_runner.cc:386] Master server non-default flags:
05:23:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/wal
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:13.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:41975
05:23:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40627/jwks.json
05:23:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data/info.pb
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/logs
05:23:13.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:13.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:13.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:13.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:13.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.959590 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.961751 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.963129 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } attempt: 4
05:23:13.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.967612 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41975: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.980786 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:14.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:14.267020 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } attempt: 5
05:23:14.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:14.270556 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41975: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:14.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:14.818188 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } attempt: 6
05:23:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:14.827373 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:41975: Network error: Client connection negotiation failed: client connection to 127.0.93.189:41975: connect: Connection refused (error 111)
05:23:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.980916 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:15.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:13.982432 456 server_base.cc:1034] running on GCE node
05:23:15.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:13.981052 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:15.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.179185 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:15.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.181718 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:15.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.183100 456 hybrid_clock.cc:648] HybridClock initialized: now 1782019395183019 us; error 99 us; skew 500 ppm
05:23:15.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.184114 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.195851 456 webserver.cc:460] Webserver started at http://127.0.93.189:34709/ using document root <none> and password file <none>
05:23:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.196959 456 fs_manager.cc:359] Metadata directory not provided
05:23:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.197237 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.197814 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:15.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.203342 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data/instance:
05:23:15.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a90813f40e9f4515ab9d9bd40ac08683"
05:23:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:15 on dist-test-slave-n3h2"
05:23:15.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.204566 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/wal/instance:
05:23:15.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a90813f40e9f4515ab9d9bd40ac08683"
05:23:15.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:15 on dist-test-slave-n3h2"
05:23:15.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.211879 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
05:23:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.217136 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:15.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.218293 456 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.218564 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/wal
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a90813f40e9f4515ab9d9bd40ac08683"
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:15 on dist-test-slave-n3h2"
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.219117 456 fs_report.cc:383] FS layout report
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/wal
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/wal
05:23:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data/data
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.265604 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:15.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.266957 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:15.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.267354 456 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.339704 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:41975
05:23:15.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.339792 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:41975 every 8 connection(s)
05:23:15.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.342373 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/data/info.pb
05:23:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.347770 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:23:15.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.350569 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 456
05:23:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.351012 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-1/wal/instance
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.353901 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/wal
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/logs
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data/info.pb
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:46309
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:15.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40627/jwks.json with env {}
05:23:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.363118 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } has no permanent_uuid. Determining permanent_uuid...
05:23:15.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.386547 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } has no permanent_uuid. Determining permanent_uuid...
05:23:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.402964 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } has no permanent_uuid. Determining permanent_uuid...
05:23:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.405534 481 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:15.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.409652 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:15.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.459090 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 1
05:23:15.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.462991 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:15.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.531426 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 2
05:23:15.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.535877 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:15.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.627597 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 3
05:23:15.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.631215 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.660583 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.661098 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.661527 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.690657 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.690920 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.691111 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.691360 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.691542 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.691672 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40627/jwks.json
05:23:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.726224 531 master_runner.cc:386] Master server non-default flags:
05:23:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/wal
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:46309
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40627/jwks.json
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data/info.pb
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/logs
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:15.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.727447 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:15.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.729450 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.750007 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.774580 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 4
05:23:15.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.778045 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:15.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.887778 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } attempt: 7
05:23:15.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.902858 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } has no permanent_uuid. Determining permanent_uuid...
05:23:15.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.909126 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:15.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.967518 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 1
05:23:15.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.971285 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:16.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.052889 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 2
05:23:16.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:16.056751 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.077590 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 5
05:23:16.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:16.081760 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:16.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.142311 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 3
05:23:16.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:16.146909 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:16.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.287317 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 4
05:23:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:16.291584 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:16.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.574962 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 5
05:23:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:16.579021 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:16.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.629150 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 6
05:23:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:16.636626 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46309: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46309: connect: Connection refused (error 111)
05:23:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.750989 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:15.752007 531 server_base.cc:1034] running on GCE node
05:23:16.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:15.751647 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:16.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.964931 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.967875 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:16.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.969281 531 hybrid_clock.cc:648] HybridClock initialized: now 1782019396969216 us; error 92 us; skew 500 ppm
05:23:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.970048 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.977495 531 webserver.cc:460] Webserver started at http://127.0.93.188:40481/ using document root <none> and password file <none>
05:23:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.978360 531 fs_manager.cc:359] Metadata directory not provided
05:23:16.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.978541 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:16.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.978927 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:16.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.983521 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data/instance:
05:23:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "397284d732a04d1b82d89877bd596147"
05:23:16.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:16 on dist-test-slave-n3h2"
05:23:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.984848 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/wal/instance:
05:23:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "397284d732a04d1b82d89877bd596147"
05:23:16.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:16 on dist-test-slave-n3h2"
05:23:16.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:16.994172 531 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
05:23:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.001469 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.003985 531 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
05:23:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.004338 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/wal
05:23:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "397284d732a04d1b82d89877bd596147"
05:23:17.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:16 on dist-test-slave-n3h2"
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.004695 531 fs_report.cc:383] FS layout report
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/wal
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/wal
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data/data
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:17.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.048477 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:17.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.050194 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:17.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.050596 531 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:17.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.123581 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:46309
05:23:17.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.123677 606 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:46309 every 8 connection(s)
05:23:17.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.126467 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/data/info.pb
05:23:17.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.131418 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 6
05:23:17.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.131806 607 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:23:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.131904 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 531
05:23:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.132419 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/master-2/wal/instance
05:23:17.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.164443 607 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } has no permanent_uuid. Determining permanent_uuid...
05:23:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.186792 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: Bootstrap starting.
05:23:17.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.193890 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: Neither blocks nor log segments found. Creating new log.
05:23:17.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.195353 607 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } has no permanent_uuid. Determining permanent_uuid...
05:23:17.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.196158 454 log.cc:824] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: Log is configured to *not* fsync() on all Append() calls
05:23:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.202677 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: No bootstrap required, opened a new log
05:23:17.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.208128 607 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } has no permanent_uuid. Determining permanent_uuid...
05:23:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.229207 607 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147: Bootstrap starting.
05:23:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.229854 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.230382 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:23:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.230571 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fbdd71e477214e53847b9cf6dbb81c1d, State: Initialized, Role: FOLLOWER
05:23:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.231218 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.234349 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:17.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.235461 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [sys.catalog]: This master's current role is: FOLLOWER
05:23:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.236967 607 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147: Neither blocks nor log segments found. Creating new log.
05:23:17.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.238094 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [sys.catalog]: configured and running, proceeding with master startup.
05:23:17.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.239631 607 log.cc:824] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147: Log is configured to *not* fsync() on all Append() calls
05:23:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.245419 607 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147: No bootstrap required, opened a new log
05:23:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.272260 607 raft_consensus.cc:348] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.273450 607 raft_consensus.cc:374] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:23:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.273957 607 raft_consensus.cc:729] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 397284d732a04d1b82d89877bd596147, State: Initialized, Role: FOLLOWER
05:23:17.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.275084 607 consensus_queue.cc:260] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:17.275966 630 catalog_manager.cc:1558] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:17.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:17.276500 630 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:17.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.281268 632 sys_catalog.cc:455] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:17.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.282291 632 sys_catalog.cc:458] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: This master's current role is: FOLLOWER
05:23:17.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.283095 607 sys_catalog.cc:564] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: configured and running, proceeding with master startup.
05:23:17.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:17.305498 561 tablet.cc:2369] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:23:17.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.272565 386 master_runner.cc:186] Error getting master registration for 127.0.93.189:41975: OK, instance_id { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" instance_seqno: 1782019395318992 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:23:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.307899 386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.317139 632 raft_consensus.cc:482] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:23:17.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.318032 632 raft_consensus.cc:504] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.321064 632 leader_election.cc:288] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fbdd71e477214e53847b9cf6dbb81c1d (127.0.93.190:40373), a90813f40e9f4515ab9d9bd40ac08683 (127.0.93.189:41975)
05:23:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.322436 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "397284d732a04d1b82d89877bd596147" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" is_pre_election: true
05:23:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.323302 429 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 397284d732a04d1b82d89877bd596147 in term 0.
05:23:17.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.322567 643 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
05:23:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.324491 558 leader_election.cc:302] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 397284d732a04d1b82d89877bd596147, fbdd71e477214e53847b9cf6dbb81c1d; no voters:
05:23:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.325502 632 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Leader pre-election won for term 1
05:23:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.326207 632 raft_consensus.cc:482] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:23:17.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.326785 632 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 0 FOLLOWER]: Advancing to term 1
05:23:17.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.334295 632 raft_consensus.cc:504] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.336318 632 leader_election.cc:288] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [CANDIDATE]: Term 1 election: Requested vote from peers fbdd71e477214e53847b9cf6dbb81c1d (127.0.93.190:40373), a90813f40e9f4515ab9d9bd40ac08683 (127.0.93.189:41975)
05:23:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.338405 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "397284d732a04d1b82d89877bd596147" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fbdd71e477214e53847b9cf6dbb81c1d"
05:23:17.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.338950 429 raft_consensus.cc:3037] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [term 0 FOLLOWER]: Advancing to term 1
05:23:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.353296 429 raft_consensus.cc:2454] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 397284d732a04d1b82d89877bd596147 in term 1.
05:23:17.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.355274 558 leader_election.cc:302] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 397284d732a04d1b82d89877bd596147, fbdd71e477214e53847b9cf6dbb81c1d; no voters:
05:23:17.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.356216 632 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 1 FOLLOWER]: Leader election won for term 1
05:23:17.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.368673 632 raft_consensus.cc:686] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [term 1 LEADER]: Becoming Leader. State: Replica: 397284d732a04d1b82d89877bd596147, State: Running, Role: LEADER
05:23:17.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.369612 632 consensus_queue.cc:237] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.333194 531 master_runner.cc:186] Error getting master registration for 127.0.93.189:41975: OK, instance_id { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" instance_seqno: 1782019395318992 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:23:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.403506 644 sys_catalog.cc:455] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 397284d732a04d1b82d89877bd596147. Latest consensus state: current_term: 1 leader_uuid: "397284d732a04d1b82d89877bd596147" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:17.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.407001 644 sys_catalog.cc:458] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: This master's current role is: LEADER
05:23:17.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.407001 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:17.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.409817 646 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:23:17.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.425525 646 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:23:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:17.456132 408 tablet.cc:2369] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:23:17.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.459473 429 raft_consensus.cc:1261] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [term 1 FOLLOWER]: Refusing update from remote peer 397284d732a04d1b82d89877bd596147: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:23:17.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.461309 644 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:23:17.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.510349 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 397284d732a04d1b82d89877bd596147. Latest consensus state: current_term: 1 leader_uuid: "397284d732a04d1b82d89877bd596147" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:17.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.511085 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [sys.catalog]: This master's current role is: FOLLOWER
05:23:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.515527 648 mvcc.cc:205] Tried to move back new op lower bound from 7299151451971792896 to 7299151451641765888. Current Snapshot: MvccSnapshot[applied={T|T < 7299151451971792896}]
05:23:17.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.525337 644 sys_catalog.cc:455] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "397284d732a04d1b82d89877bd596147" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:17.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.526224 644 sys_catalog.cc:458] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: This master's current role is: LEADER
05:23:17.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.530402 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "397284d732a04d1b82d89877bd596147" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:17.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.531584 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d [sys.catalog]: This master's current role is: FOLLOWER
05:23:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.538825 646 catalog_manager.cc:1347] Generated new cluster ID: 2a1bcdb0d5904f5fa5fbd3f4f7c8f69d
05:23:17.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.539175 646 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:23:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.573683 646 catalog_manager.cc:1370] Generated new certificate authority record
05:23:17.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.581492 646 catalog_manager.cc:1504] Loading token signing keys...
05:23:17.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.631227 646 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147: Generated new TSK 0
05:23:17.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.632381 646 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.661975 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/wal
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:17.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/logs
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data/info.pb
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:17.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:23:17.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.696961 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } attempt: 7
05:23:17.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.747049 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683: Bootstrap starting.
05:23:17.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.766826 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683: Neither blocks nor log segments found. Creating new log.
05:23:17.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.772361 529 log.cc:824] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683: Log is configured to *not* fsync() on all Append() calls
05:23:17.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.793913 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683: No bootstrap required, opened a new log
05:23:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.870121 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.872545 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:23:17.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.873551 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a90813f40e9f4515ab9d9bd40ac08683, State: Initialized, Role: FOLLOWER
05:23:17.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.876224 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.892777 656 raft_consensus.cc:482] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:23:17.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.893529 656 raft_consensus.cc:504] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } }
05:23:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.897504 656 leader_election.cc:288] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fbdd71e477214e53847b9cf6dbb81c1d (127.0.93.190:40373), 397284d732a04d1b82d89877bd596147 (127.0.93.188:46309)
05:23:17.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.898192 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a90813f40e9f4515ab9d9bd40ac08683" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" is_pre_election: true
05:23:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.899745 582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a90813f40e9f4515ab9d9bd40ac08683" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "397284d732a04d1b82d89877bd596147" is_pre_election: true
05:23:17.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.901324 481 leader_election.cc:302] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a90813f40e9f4515ab9d9bd40ac08683; no voters: 397284d732a04d1b82d89877bd596147, fbdd71e477214e53847b9cf6dbb81c1d
05:23:17.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.903067 656 raft_consensus.cc:2726] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:23:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.905000 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "397284d732a04d1b82d89877bd596147" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a90813f40e9f4515ab9d9bd40ac08683" is_pre_election: true
05:23:17.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.905866 502 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 0 FOLLOWER]: Advancing to term 1
05:23:17.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.905031 503 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "397284d732a04d1b82d89877bd596147" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a90813f40e9f4515ab9d9bd40ac08683"
05:23:17.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.915463 502 raft_consensus.cc:1261] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [term 1 FOLLOWER]: Refusing update from remote peer 397284d732a04d1b82d89877bd596147: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:23:17.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.904862 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.918023 632 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:23:17.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.916688 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [sys.catalog]: configured and running, proceeding with master startup.
05:23:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.917719 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [sys.catalog]: This master's current role is: FOLLOWER
05:23:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:17.977202 659 mvcc.cc:205] Tried to move back new op lower bound from 7299151451971792896 to 7299151451641765888. Current Snapshot: MvccSnapshot[applied={T|T < 7299151451971792896}]
05:23:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.000798 656 sys_catalog.cc:455] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "397284d732a04d1b82d89877bd596147" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.001437 656 sys_catalog.cc:458] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [sys.catalog]: This master's current role is: FOLLOWER
05:23:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.000115 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 397284d732a04d1b82d89877bd596147. Latest consensus state: current_term: 1 leader_uuid: "397284d732a04d1b82d89877bd596147" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:18.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.002911 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683 [sys.catalog]: This master's current role is: FOLLOWER
05:23:18.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.078918 672 catalog_manager.cc:1259] Loaded cluster ID: 2a1bcdb0d5904f5fa5fbd3f4f7c8f69d
05:23:18.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.079207 672 catalog_manager.cc:1552] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683: loading cluster ID for follower catalog manager: success
05:23:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.093271 672 catalog_manager.cc:1574] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683: acquiring CA information for follower catalog manager: success
05:23:18.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.099053 672 catalog_manager.cc:1602] T 00000000000000000000000000000000 P a90813f40e9f4515ab9d9bd40ac08683: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:23:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:18.181572 653 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:18.182512 653 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:18.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:18.183462 653 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:18.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:18.240451 653 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:18.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:18.242117 653 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
05:23:18.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.282552 630 catalog_manager.cc:1259] Loaded cluster ID: 2a1bcdb0d5904f5fa5fbd3f4f7c8f69d
05:23:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.283125 630 catalog_manager.cc:1552] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: loading cluster ID for follower catalog manager: success
05:23:18.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.300833 630 catalog_manager.cc:1574] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: acquiring CA information for follower catalog manager: success
05:23:18.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.306985 630 catalog_manager.cc:1602] T 00000000000000000000000000000000 P fbdd71e477214e53847b9cf6dbb81c1d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.324918 653 tablet_server_runner.cc:78] Tablet server non-default flags:
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/wal
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:18.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data/info.pb
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/logs
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.333483 653 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.335863 653 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:18.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.371225 632 sys_catalog.cc:455] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "397284d732a04d1b82d89877bd596147" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fbdd71e477214e53847b9cf6dbb81c1d" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 40373 } } peers { permanent_uuid: "a90813f40e9f4515ab9d9bd40ac08683" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 41975 } } peers { permanent_uuid: "397284d732a04d1b82d89877bd596147" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46309 } } }
05:23:18.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:18.373413 632 sys_catalog.cc:458] T 00000000000000000000000000000000 P 397284d732a04d1b82d89877bd596147 [sys.catalog]: This master's current role is: LEADER
05:23:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:18.377719 681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:19.758440 679 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 653
05:23:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:21.270609 653 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.891s user 0.798s sys 1.329s
05:23:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:21.271804 653 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.893s user 0.798s sys 1.329s
05:23:21.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:21.273600 682 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2893 milliseconds
05:23:21.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:21.273768 685 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:21.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.274556 653 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:23:21.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:18.382670 680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.281332 653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:21.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.284081 653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:21.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.285748 653 hybrid_clock.cc:648] HybridClock initialized: now 1782019401285711 us; error 44 us; skew 500 ppm
05:23:21.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.286758 653 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.298381 653 webserver.cc:460] Webserver started at http://127.0.93.129:39899/ using document root <none> and password file <none>
05:23:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.299733 653 fs_manager.cc:359] Metadata directory not provided
05:23:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.300004 653 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:21.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.300550 653 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.307348 653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data/instance:
05:23:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bd2e1acd8624713b9d7cfd14c2d70ab"
05:23:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:21 on dist-test-slave-n3h2"
05:23:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.308813 653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/wal/instance:
05:23:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bd2e1acd8624713b9d7cfd14c2d70ab"
05:23:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:21 on dist-test-slave-n3h2"
05:23:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.317978 653 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
05:23:21.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.326546 692 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.327841 653 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
05:23:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.328292 653 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/wal
05:23:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bd2e1acd8624713b9d7cfd14c2d70ab"
05:23:21.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:21 on dist-test-slave-n3h2"
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.328727 653 fs_report.cc:383] FS layout report
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/wal
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/wal
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data/data
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:21.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:21.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.418061 653 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.419808 653 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.420368 653 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:21.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.424078 653 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:23:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.429193 653 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:23:21.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.429414 653 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:23:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.429744 653 ts_tablet_manager.cc:610] Registered 0 tablets
05:23:21.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.429924 653 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:23:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.701135 653 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:37217
05:23:21.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.701226 805 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:37217 every 8 connection(s)
05:23:21.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.704128 653 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/data/info.pb
05:23:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.714870 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 653
05:23:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.715619 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-0/wal/instance
05:23:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.743896 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/wal
05:23:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data
05:23:21.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/logs
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data/info.pb
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:21.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:21.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:23:21.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.764892 806 heartbeater.cc:347] Connected to a master server at 127.0.93.188:46309
05:23:21.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.765415 806 heartbeater.cc:464] Registering TS with master...
05:23:21.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.766511 806 heartbeater.cc:511] Master 127.0.93.188:46309 requested a full tablet report, sending...
05:23:21.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.769336 572 ts_manager.cc:194] Registered new tserver with Master: 4bd2e1acd8624713b9d7cfd14c2d70ab (127.0.93.129:37217)
05:23:21.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.771942 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:52685
05:23:21.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.796818 807 heartbeater.cc:347] Connected to a master server at 127.0.93.189:41975
05:23:21.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.797186 807 heartbeater.cc:464] Registering TS with master...
05:23:21.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.797991 807 heartbeater.cc:511] Master 127.0.93.189:41975 requested a full tablet report, sending...
05:23:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.801523 493 ts_manager.cc:194] Registered new tserver with Master: 4bd2e1acd8624713b9d7cfd14c2d70ab (127.0.93.129:37217)
05:23:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.802855 811 heartbeater.cc:347] Connected to a master server at 127.0.93.190:40373
05:23:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.803211 811 heartbeater.cc:464] Registering TS with master...
05:23:21.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.803953 811 heartbeater.cc:511] Master 127.0.93.190:40373 requested a full tablet report, sending...
05:23:21.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:21.806668 419 ts_manager.cc:194] Registered new tserver with Master: 4bd2e1acd8624713b9d7cfd14c2d70ab (127.0.93.129:37217)
05:23:22.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.170403 810 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:22.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.170943 810 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:22.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.171552 810 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:22.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.204653 810 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.205590 810 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:22.240833 810 tablet_server_runner.cc:78] Tablet server non-default flags:
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/wal
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data/info.pb
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/logs
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:22.242354 810 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:22.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:22.244032 810 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:22.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.264169 823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:22.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:22.777365 806 heartbeater.cc:503] Master 127.0.93.188:46309 was elected leader, sending a full tablet report...
05:23:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.264245 826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:23.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:22.266857 810 server_base.cc:1034] running on GCE node
05:23:23.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:22.264410 824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:23.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.455402 810 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:23.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.457787 810 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:23.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.459174 810 hybrid_clock.cc:648] HybridClock initialized: now 1782019403459125 us; error 59 us; skew 500 ppm
05:23:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.459915 810 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:23.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.466012 810 webserver.cc:460] Webserver started at http://127.0.93.130:34427/ using document root <none> and password file <none>
05:23:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.466842 810 fs_manager.cc:359] Metadata directory not provided
05:23:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.467015 810 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:23.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.467377 810 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:23.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.471956 810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data/instance:
05:23:23.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c038ec7f761c468290a867dbcd1c35d6"
05:23:23.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:23 on dist-test-slave-n3h2"
05:23:23.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.473037 810 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/wal/instance:
05:23:23.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c038ec7f761c468290a867dbcd1c35d6"
05:23:23.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:23 on dist-test-slave-n3h2"
05:23:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.481632 810 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
05:23:23.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.488734 834 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.489966 810 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:23:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.490314 810 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/wal
05:23:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c038ec7f761c468290a867dbcd1c35d6"
05:23:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:23 on dist-test-slave-n3h2"
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.490650 810 fs_report.cc:383] FS layout report
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/wal
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/wal
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data/data
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.524724 810 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:23.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.526352 810 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:23.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.526868 810 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:23.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.529775 810 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:23:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.533910 810 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:23:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.534170 810 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:23:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.534389 810 ts_tablet_manager.cc:610] Registered 0 tablets
05:23:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.534514 810 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:23:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.680521 810 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:40763
05:23:23.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.680610 946 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:40763 every 8 connection(s)
05:23:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.682847 810 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/data/info.pb
05:23:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.686432 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 810
05:23:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.686892 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-1/wal/instance
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.703645 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/wal
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/logs
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data/info.pb
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:23.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:23:23.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.754061 948 heartbeater.cc:347] Connected to a master server at 127.0.93.189:41975
05:23:23.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.754568 948 heartbeater.cc:464] Registering TS with master...
05:23:23.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.755851 948 heartbeater.cc:511] Master 127.0.93.189:41975 requested a full tablet report, sending...
05:23:23.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.756618 947 heartbeater.cc:347] Connected to a master server at 127.0.93.188:46309
05:23:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.756976 947 heartbeater.cc:464] Registering TS with master...
05:23:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.757700 947 heartbeater.cc:511] Master 127.0.93.188:46309 requested a full tablet report, sending...
05:23:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.758988 949 heartbeater.cc:347] Connected to a master server at 127.0.93.190:40373
05:23:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.759277 949 heartbeater.cc:464] Registering TS with master...
05:23:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.759318 493 ts_manager.cc:194] Registered new tserver with Master: c038ec7f761c468290a867dbcd1c35d6 (127.0.93.130:40763)
05:23:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.759744 572 ts_manager.cc:194] Registered new tserver with Master: c038ec7f761c468290a867dbcd1c35d6 (127.0.93.130:40763)
05:23:23.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.759989 949 heartbeater.cc:511] Master 127.0.93.190:40373 requested a full tablet report, sending...
05:23:23.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.761592 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:60905
05:23:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:23.762820 419 ts_manager.cc:194] Registered new tserver with Master: c038ec7f761c468290a867dbcd1c35d6 (127.0.93.130:40763)
05:23:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.041337 957 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.041826 957 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:24.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.042289 957 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.072222 957 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.072974 957 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:24.105929 957 tablet_server_runner.cc:78] Tablet server non-default flags:
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:35943
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/wal
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data/info.pb
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:40373,127.0.93.189:41975,127.0.93.188:46309
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/logs
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:24.107357 957 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:24.109197 957 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:24.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.128981 965 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:24.765267 947 heartbeater.cc:503] Master 127.0.93.188:46309 was elected leader, sending a full tablet report...
05:23:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.130087 967 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:24.131150 957 server_base.cc:1034] running on GCE node
05:23:25.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:24.137794 964 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:25.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.323174 957 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:25.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.325552 957 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:25.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.326951 957 hybrid_clock.cc:648] HybridClock initialized: now 1782019405326910 us; error 34 us; skew 500 ppm
05:23:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.327901 957 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:25.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.335341 957 webserver.cc:460] Webserver started at http://127.0.93.131:45477/ using document root <none> and password file <none>
05:23:25.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.336304 957 fs_manager.cc:359] Metadata directory not provided
05:23:25.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.336530 957 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:25.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.337030 957 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.343230 957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data/instance:
05:23:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea9754f738b14f2ea7f86535fc5fa02d"
05:23:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:25 on dist-test-slave-n3h2"
05:23:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.344597 957 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/wal/instance:
05:23:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea9754f738b14f2ea7f86535fc5fa02d"
05:23:25.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:25 on dist-test-slave-n3h2"
05:23:25.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.353145 957 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
05:23:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.360144 974 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.361328 957 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.361671 957 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/wal
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea9754f738b14f2ea7f86535fc5fa02d"
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:25 on dist-test-slave-n3h2"
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.362047 957 fs_report.cc:383] FS layout report
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/wal
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/wal
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data/data
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:25.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:25.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.400764 957 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.402143 957 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:25.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.402526 957 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.405179 957 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:23:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.409243 957 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:23:25.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.409426 957 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:23:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.409616 957 ts_tablet_manager.cc:610] Registered 0 tablets
05:23:25.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.409736 957 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:23:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.559098 957 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:41645
05:23:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.559245 1086 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:41645 every 8 connection(s)
05:23:25.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.561654 957 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/data/info.pb
05:23:25.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.562395 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 957
05:23:25.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.562816 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster5877160310729450921/ts-2/wal/instance
05:23:25.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.602342 1087 heartbeater.cc:347] Connected to a master server at 127.0.93.188:46309
05:23:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.602815 1087 heartbeater.cc:464] Registering TS with master...
05:23:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.603223 1088 heartbeater.cc:347] Connected to a master server at 127.0.93.189:41975
05:23:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.603536 1088 heartbeater.cc:464] Registering TS with master...
05:23:25.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.604010 1087 heartbeater.cc:511] Master 127.0.93.188:46309 requested a full tablet report, sending...
05:23:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.604506 1088 heartbeater.cc:511] Master 127.0.93.189:41975 requested a full tablet report, sending...
05:23:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.604812 1091 heartbeater.cc:347] Connected to a master server at 127.0.93.190:40373
05:23:25.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.605099 1091 heartbeater.cc:464] Registering TS with master...
05:23:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.606241 1091 heartbeater.cc:511] Master 127.0.93.190:40373 requested a full tablet report, sending...
05:23:25.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.606725 572 ts_manager.cc:194] Registered new tserver with Master: ea9754f738b14f2ea7f86535fc5fa02d (127.0.93.131:41645)
05:23:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.608680 419 ts_manager.cc:194] Registered new tserver with Master: ea9754f738b14f2ea7f86535fc5fa02d (127.0.93.131:41645)
05:23:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.608927 572 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:55877
05:23:25.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.609306 493 ts_manager.cc:194] Registered new tserver with Master: ea9754f738b14f2ea7f86535fc5fa02d (127.0.93.131:41645)
05:23:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:25.629959 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:23:25.642 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:23:25.643 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:23:25.656 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 40373
}
bound_http_address {
host: "127.0.93.190"
port: 34459
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 41975
}
bound_http_address {
host: "127.0.93.189"
port: 34709
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 46309
}
bound_http_address {
host: "127.0.93.188"
port: 40481
}
}
}
05:23:25.662 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:23:25.665 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 37217
}
bound_http_address {
host: "127.0.93.129"
port: 39899
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 40763
}
bound_http_address {
host: "127.0.93.130"
port: 34427
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 41645
}
bound_http_address {
host: "127.0.93.131"
port: 45477
}
}
}
05:23:25.665 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:23:25.833 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:23:25.953 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task55sErp/test-tmp
05:23:25.954 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:23:25.954 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task55sErp/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:23:25.957 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:23:26.313 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:26.313 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:23:26.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:26.316475 1099 mini_oidc.cc:152] Starting JWKS server
05:23:26.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:26.320616 1099 webserver.cc:460] Webserver started at http://127.0.0.1:37717/ using document root <none> and password file <none>
05:23:26.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:26.321192 1099 mini_oidc.cc:189] Starting OIDC Discovery server
05:23:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:26.322978 1099 webserver.cc:460] Webserver started at http://127.0.0.1:33213/ using document root <none> and password file <none>
05:23:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:23:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/principal' for realm 'KRBTEST.COM',
05:23:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:23:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:26 dist-test-slave-n3h2 krb5kdc[1111](info): setting up network...
05:23:26.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:23:26.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:26 dist-test-slave-n3h2 krb5kdc[1111](info): set up 2 sockets
05:23:26.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:26 dist-test-slave-n3h2 krb5kdc[1111](info): commencing operation
05:23:26.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:23:26.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:26.613145 1087 heartbeater.cc:503] Master 127.0.93.188:46309 was elected leader, sending a full tablet report...
05:23:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.428797 1099 mini_kdc.cc:121] Time spent starting KDC: real 1.079s user 0.003s sys 0.018s
05:23:27.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:23:27.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:23:27.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:23:27.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:23:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:23:27.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:23:27.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:23:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:23:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:23:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:23:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/test-user.keytab.
05:23:27.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/test-user.keytab.
05:23:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:27 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:23:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:23:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:23:27.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:23:27Z Disabled control of system clock
05:23:27.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.254@KRBTEST.COM; defaulting to no policy
05:23:27.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:23:27.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.254@KRBTEST.COM" created.
05:23:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:23:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.254.keytab.
05:23:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.254.keytab.
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:27.625033 1099 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/logs
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/info.pb
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.254:45727
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.254
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.254.keytab
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.254
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:23:27.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.928085 1124 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:27.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.928759 1124 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.929522 1124 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:27.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.962352 1124 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:23:27.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.962612 1124 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:27.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.962980 1124 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:27.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.963294 1124 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:27.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.963601 1124 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:27.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.963899 1124 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:27.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:27.964116 1124 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37717/jwks.json
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:27.999636 1124 master_runner.cc:386] Master server non-default flags:
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.254.keytab
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.254
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.254:45727
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:23:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/info.pb
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.254
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/logs
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:28.001092 1124 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:28.002381 1124 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:28.021121 1130 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:28.023313 1133 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:28.024070 1124 server_base.cc:1034] running on GCE node
05:23:29.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:28.021167 1131 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.261495 1124 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.265132 1124 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.266592 1124 hybrid_clock.cc:648] HybridClock initialized: now 1782019409266528 us; error 78 us; skew 500 ppm
05:23:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.267443 1124 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:29 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019409, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:29.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.273391 1124 init.cc:377] Logged in from keytab as kudu/127.1.18.254@KRBTEST.COM (short username kudu)
05:23:29.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.278285 1124 webserver.cc:460] Webserver started at http://127.1.18.254:39029/ using document root <none> and password file <none>
05:23:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.279098 1124 fs_manager.cc:359] Metadata directory not provided
05:23:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.279265 1124 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.279621 1124 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.284700 1124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/instance:
05:23:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5eeec8ee53e4ddea4b534bad768bab6"
05:23:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:29 on dist-test-slave-n3h2"
05:23:29.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.285738 1124 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal/instance:
05:23:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5eeec8ee53e4ddea4b534bad768bab6"
05:23:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:29 on dist-test-slave-n3h2"
05:23:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.292800 1124 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
05:23:29.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.298180 1142 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:29.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.299213 1124 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
05:23:29.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.299486 1124 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:29.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5eeec8ee53e4ddea4b534bad768bab6"
05:23:29.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:29 on dist-test-slave-n3h2"
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.299760 1124 fs_report.cc:383] FS layout report
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/data
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.337589 1124 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:29.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.348207 1124 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.348724 1124 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.463958 1124 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.254:45727
05:23:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.464047 1193 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.254:45727 every 8 connection(s)
05:23:29.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.466629 1124 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/info.pb
05:23:29.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.471024 1099 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 1124
05:23:29.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.471544 1099 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal/instance
05:23:29.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.472539 1194 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:23:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.489425 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } has no permanent_uuid. Determining permanent_uuid...
05:23:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.253@KRBTEST.COM; defaulting to no policy
05:23:29.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:23:29.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.253@KRBTEST.COM" created.
05:23:29.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:29 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019409, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.254@KRBTEST.COM for kudu/127.1.18.254@KRBTEST.COM
05:23:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:23:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.253.keytab.
05:23:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.253.keytab.
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.520895 1099 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/logs
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/info.pb
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.253:38429
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.253
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.253.keytab
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.253
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:23:29.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.529793 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
05:23:29.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.532620 1146 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:29.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.537223 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.253:38429: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:29.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.586756 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } attempt: 1
05:23:29.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.590229 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.253:38429: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:29.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.658659 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } attempt: 2
05:23:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.662408 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.253:38429: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.753765 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } attempt: 3
05:23:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.757344 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.253:38429: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:29.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.841651 1200 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:29.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.842285 1200 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.842947 1200 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.873700 1200 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:23:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.873973 1200 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.874356 1200 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.874670 1200 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.874981 1200 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.875286 1200 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.875506 1200 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37717/jwks.json
05:23:29.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.900728 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } attempt: 4
05:23:29.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.904613 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.253:38429: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.913273 1200 master_runner.cc:386] Master server non-default flags:
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:23:29.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.253.keytab
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.253
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.253:38429
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/info.pb
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.253
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/logs
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:29.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.914695 1200 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:29.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.915988 1200 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.935367 1207 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:30.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:30.203958 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } attempt: 5
05:23:30.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:30.208868 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.253:38429: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:30.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:30.408443 802 debug-util.cc:398] Leaking SignalData structure 0x7b08000afbc0 after lost signal to thread 674
05:23:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:30.409345 802 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb100 after lost signal to thread 805
05:23:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:30.756269 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } attempt: 6
05:23:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:30.767745 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.253:38429: Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:29.937515 1200 server_base.cc:1034] running on GCE node
05:23:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.936584 1210 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:29.935396 1208 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:31.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.211066 1200 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:31.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.214231 1200 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:31.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.215652 1200 hybrid_clock.cc:648] HybridClock initialized: now 1782019411215597 us; error 78 us; skew 500 ppm
05:23:31.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.216466 1200 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:31.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:31 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019411, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:31.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.223209 1200 init.cc:377] Logged in from keytab as kudu/127.1.18.253@KRBTEST.COM (short username kudu)
05:23:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.229586 1200 webserver.cc:460] Webserver started at http://127.1.18.253:36195/ using document root <none> and password file <none>
05:23:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.230491 1200 fs_manager.cc:359] Metadata directory not provided
05:23:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.230713 1200 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:31.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.231189 1200 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.235945 1200 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/instance:
05:23:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d883a5df4994fda874847144e000593"
05:23:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:31 on dist-test-slave-n3h2"
05:23:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.237083 1200 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal/instance:
05:23:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d883a5df4994fda874847144e000593"
05:23:31.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:31 on dist-test-slave-n3h2"
05:23:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.244843 1200 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
05:23:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.250830 1220 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.251930 1200 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
05:23:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.252246 1200 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d883a5df4994fda874847144e000593"
05:23:31.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:31 on dist-test-slave-n3h2"
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.252592 1200 fs_report.cc:383] FS layout report
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/data
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:31.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.299283 1200 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.305373 1200 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:31.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.305891 1200 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.413208 1200 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.253:38429
05:23:31.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.413237 1271 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.253:38429 every 8 connection(s)
05:23:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.417282 1200 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/info.pb
05:23:31.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.424070 1272 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:23:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.427275 1099 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 1200
05:23:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.427703 1099 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal/instance
05:23:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.439644 1272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } has no permanent_uuid. Determining permanent_uuid...
05:23:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.18.252@KRBTEST.COM; defaulting to no policy
05:23:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:23:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.18.252@KRBTEST.COM" created.
05:23:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:31 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019411, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.253@KRBTEST.COM for kudu/127.1.18.254@KRBTEST.COM
05:23:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:23:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.252.keytab.
05:23:31.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.18.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.252.keytab.
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.485905 1099 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/logs
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/info.pb
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.252:44961
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.252
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.252.keytab
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.252
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:23:31.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.492702 1272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
05:23:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:31 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019411, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.253@KRBTEST.COM for kudu/127.1.18.253@KRBTEST.COM
05:23:31.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.523406 1272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } has no permanent_uuid. Determining permanent_uuid...
05:23:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.525794 1223 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.529848 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.579309 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 1
05:23:31.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.583519 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.651944 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 2
05:23:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.657047 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.748458 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 3
05:23:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.752934 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.828209 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } attempt: 7
05:23:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.837234 1278 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:31.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.838148 1278 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:31.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.840006 1278 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:31.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:31 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019409, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.254@KRBTEST.COM for kudu/127.1.18.253@KRBTEST.COM
05:23:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.859987 1194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } has no permanent_uuid. Determining permanent_uuid...
05:23:31.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.866180 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.879019 1278 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:23:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.879285 1278 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.879676 1278 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.879989 1278 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.880293 1278 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:31.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.880605 1278 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.880857 1278 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37717/jwks.json
05:23:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.896330 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 4
05:23:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.901162 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.918694 1278 master_runner.cc:386] Master server non-default flags:
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.252.keytab
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.252
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.252:44961
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:31.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/info.pb
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.252
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/logs
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:31.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.920130 1278 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:31.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.921423 1278 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:31.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:31.924582 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 1
05:23:31.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.928493 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:31.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.948135 1289 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:32.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:32.009889 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 2
05:23:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:32.015216 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:32.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:32.100633 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 3
05:23:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:32.106514 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:32.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:32.200595 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 5
05:23:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:32.205755 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:32.247061 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 4
05:23:32.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:32.256273 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:32.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:32.539812 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 5
05:23:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:32.550529 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:32.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:32.753168 1272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 6
05:23:32.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:32.763154 1272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:33.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.103052 1194 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } attempt: 6
05:23:33.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:33.111613 1194 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.18.252:44961: Network error: Client connection negotiation failed: client connection to 127.1.18.252:44961: connect: Connection refused (error 111)
05:23:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.951500 1292 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:33.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:31.948776 1290 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:33.278823 1291 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1329 milliseconds
05:23:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.278944 1278 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:23:33.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.280035 1278 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.283010 1278 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:33.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.284407 1278 hybrid_clock.cc:648] HybridClock initialized: now 1782019413284377 us; error 44 us; skew 500 ppm
05:23:33.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.285286 1278 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:33.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019413, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:33.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.291239 1278 init.cc:377] Logged in from keytab as kudu/127.1.18.252@KRBTEST.COM (short username kudu)
05:23:33.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.296307 1278 webserver.cc:460] Webserver started at http://127.1.18.252:32985/ using document root <none> and password file <none>
05:23:33.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.297156 1278 fs_manager.cc:359] Metadata directory not provided
05:23:33.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.297340 1278 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:33.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.297698 1278 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:23:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.302238 1278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/instance:
05:23:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1549ff959894c3288479c3861335599"
05:23:33.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:33 on dist-test-slave-n3h2"
05:23:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.303315 1278 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal/instance:
05:23:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1549ff959894c3288479c3861335599"
05:23:33.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:33 on dist-test-slave-n3h2"
05:23:33.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.310215 1278 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.003s
05:23:33.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.315479 1306 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:33.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.316509 1278 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.316833 1278 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1549ff959894c3288479c3861335599"
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:33 on dist-test-slave-n3h2"
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.317191 1278 fs_report.cc:383] FS layout report
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/data
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:33.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:33.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:33.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.352181 1278 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.358933 1278 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.359396 1278 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:33.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.438601 1278 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.252:44961
05:23:33.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.438658 1357 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.252:44961 every 8 connection(s)
05:23:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.441318 1278 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/info.pb
05:23:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.441903 1099 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 1278
05:23:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.442377 1099 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal/instance
05:23:33.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.447750 1358 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:23:33.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.468112 1358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } has no permanent_uuid. Determining permanent_uuid...
05:23:33.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.252@KRBTEST.COM
05:23:33.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019413, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.252@KRBTEST.COM for kudu/127.1.18.254@KRBTEST.COM
05:23:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.501088 1358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } has no permanent_uuid. Determining permanent_uuid...
05:23:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019413, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.252@KRBTEST.COM for kudu/127.1.18.253@KRBTEST.COM
05:23:33.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.253@KRBTEST.COM
05:23:33.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.528813 1358 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } has no permanent_uuid. Determining permanent_uuid...
05:23:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019413, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.252@KRBTEST.COM for kudu/127.1.18.252@KRBTEST.COM
05:23:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019407, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.254@KRBTEST.COM
05:23:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.562680 1099 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:23:33.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.562688 1358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Bootstrap starting.
05:23:33.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.569801 1358 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Neither blocks nor log segments found. Creating new log.
05:23:33.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.571796 1358 log.cc:824] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Log is configured to *not* fsync() on all Append() calls
05:23:33.574 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:33.575 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:23:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.578456 1358 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: No bootstrap required, opened a new log
05:23:33.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:23:33.589 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:23:33.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.18.254"
port: 45727
}
bound_http_address {
host: "127.1.18.254"
port: 39029
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.18.253"
port: 38429
}
bound_http_address {
host: "127.1.18.253"
port: 36195
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.18.252"
port: 44961
}
bound_http_address {
host: "127.1.18.252"
port: 32985
}
}
}
05:23:33.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:23:33.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:33.592 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:23:33.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:23:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.603885 1358 raft_consensus.cc:348] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:33.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.604564 1358 raft_consensus.cc:374] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:23:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.604854 1358 raft_consensus.cc:729] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f1549ff959894c3288479c3861335599, State: Initialized, Role: FOLLOWER
05:23:33.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.605621 1358 consensus_queue.cc:260] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:33.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.608619 1368 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } } }
05:23:33.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.609511 1368 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [sys.catalog]: This master's current role is: FOLLOWER
05:23:33.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.610190 1358 sys_catalog.cc:564] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [sys.catalog]: configured and running, proceeding with master startup.
05:23:33.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:33.615581 1312 tablet.cc:2369] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:23:33.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019413, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:33.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:23:33.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:33.631 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc
05:23:33.637 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:23:33.639 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc
05:23:33.640 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:23:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:33.641567 1380 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:33.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:33.641887 1380 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:33.646 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:23:33.648 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:23:33.649 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:23:33.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.637171 1278 master_runner.cc:186] Error getting master registration for 127.1.18.254:45727: OK, instance_id { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" instance_seqno: 1782019409443461 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:23:33.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.671945 1278 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:33.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:33 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019413, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:33.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:23:33.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:33.688 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc
05:23:33.689 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:23:33.690 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.253:38429
05:23:33.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:23:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.692555 1099 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 1200
05:23:33.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:33.719 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.254:45727
05:23:33.719 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:23:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.720147 1099 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 1124
05:23:33.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:33.751 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.18.252:44961
05:23:33.751 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:23:33.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.752106 1099 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 1278
05:23:33.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:33.775 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.253:38429
05:23:33.776 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:33.778566 1099 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/logs
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/info.pb
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.253:38429
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.253
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36195
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.253.keytab
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.253
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:33.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:23:34.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.086371 1384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.086882 1384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:34.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.087455 1384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:34.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.118520 1384 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:23:34.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.118734 1384 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.118988 1384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.119179 1384 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.119400 1384 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.119630 1384 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.119776 1384 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37717/jwks.json
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:34.169013 1384 master_runner.cc:386] Master server non-default flags:
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.253.keytab
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.253
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.253:38429
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:34.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/info.pb
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.253
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36195
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/logs
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:34.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:34.170084 1384 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:34.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:34.171012 1384 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.187742 1393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.187762 1392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:34.188721 1395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:34.190124 1384 server_base.cc:1034] running on GCE node
05:23:35.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.401518 1384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:35.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.404162 1384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:35.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.405577 1384 hybrid_clock.cc:648] HybridClock initialized: now 1782019415405544 us; error 37 us; skew 500 ppm
05:23:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.406375 1384 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:35.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:35 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019415, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:35.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.412343 1384 init.cc:377] Logged in from keytab as kudu/127.1.18.253@KRBTEST.COM (short username kudu)
05:23:35.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.417048 1384 webserver.cc:460] Webserver started at http://127.1.18.253:36195/ using document root <none> and password file <none>
05:23:35.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.417932 1384 fs_manager.cc:359] Metadata directory not provided
05:23:35.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.418175 1384 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:35.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.425851 1384 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.002s
05:23:35.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.429993 1403 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:35.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.430907 1384 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
05:23:35.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.431233 1384 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:35.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d883a5df4994fda874847144e000593"
05:23:35.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:31 on dist-test-slave-n3h2"
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.433544 1384 fs_report.cc:383] FS layout report
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/data
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:35.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.519770 1384 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.525336 1384 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:35.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.525875 1384 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:35.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.601682 1384 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.253:38429
05:23:35.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.601883 1454 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.253:38429 every 8 connection(s)
05:23:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.604688 1384 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/data/info.pb
05:23:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.611177 1099 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 1384
05:23:35.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:35.612 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.254:45727
05:23:35.612 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:23:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.613462 1099 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/logs
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/info.pb
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.254:45727
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.254
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39029
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.254.keytab
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.254
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.615389 1455 sys_catalog.cc:263] Verifying existing consensus state
05:23:35.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 05:23:35.616321 1455 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:23:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.616799 1384 master.cc:537] Master@127.1.18.253:38429 shutting down...
05:23:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.679139 1384 master.cc:552] Master@127.1.18.253:38429 shutdown complete.
05:23:35.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:23:35.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.922747 1457 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:35.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.923241 1457 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:35.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.923859 1457 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:35.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.954800 1457 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:23:35.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.955018 1457 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:35.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.955269 1457 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:35.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.955515 1457 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:35.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.955775 1457 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.955972 1457 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:35.956110 1457 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37717/jwks.json
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.993009 1457 master_runner.cc:386] Master server non-default flags:
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.254.keytab
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.254
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.254:45727
05:23:35.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/info.pb
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.254
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39029
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/logs
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:35.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:35.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:35.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.994055 1457 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:35.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:35.995080 1457 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:36.012329 1464 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:36.012545 1465 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:36.013947 1457 server_base.cc:1034] running on GCE node
05:23:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:36.016456 1467 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:37.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.223078 1457 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:37.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.225729 1457 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:37.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.227094 1457 hybrid_clock.cc:648] HybridClock initialized: now 1782019417227057 us; error 51 us; skew 500 ppm
05:23:37.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.227866 1457 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:37.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:37 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019417, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:37.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.239660 1457 init.cc:377] Logged in from keytab as kudu/127.1.18.254@KRBTEST.COM (short username kudu)
05:23:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.244899 1457 webserver.cc:460] Webserver started at http://127.1.18.254:39029/ using document root <none> and password file <none>
05:23:37.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.245781 1457 fs_manager.cc:359] Metadata directory not provided
05:23:37.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.245978 1457 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.253944 1457 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
05:23:37.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.258011 1476 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.259088 1457 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
05:23:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.259352 1457 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5eeec8ee53e4ddea4b534bad768bab6"
05:23:37.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:29 on dist-test-slave-n3h2"
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.261584 1457 fs_report.cc:383] FS layout report
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/data
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:37.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:37.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.306267 1457 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.311537 1457 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.311923 1457 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.391112 1457 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.254:45727
05:23:37.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.391227 1527 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.254:45727 every 8 connection(s)
05:23:37.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.393625 1457 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/data/info.pb
05:23:37.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.400864 1099 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 1457
05:23:37.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:37.402 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.18.252:44961
05:23:37.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:23:37.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.403542 1528 sys_catalog.cc:263] Verifying existing consensus state
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.404018 1099 external_mini_cluster.cc:1326] Running /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task55sErp/build/tsan/bin/kudu
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/logs
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/info.pb
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:23:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.252:44961
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.252
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32985
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.252.keytab
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.252
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:23:37.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 05:23:37.404484 1528 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:23:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.404807 1457 master.cc:537] Master@127.1.18.254:45727 shutting down...
05:23:37.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.459883 1457 master.cc:552] Master@127.1.18.254:45727 shutdown complete.
05:23:37.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:23:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.711742 1530 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:23:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.712313 1530 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:23:37.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.712888 1530 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:23:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.743691 1530 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:23:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.743899 1530 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:23:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.744148 1530 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:23:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.744362 1530 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:23:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.744580 1530 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:23:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.744794 1530 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:23:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.744936 1530 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37717/jwks.json
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.780000 1530 master_runner.cc:386] Master server non-default flags:
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.18.212:43253
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.18.254:45727,127.1.18.253:38429,127.1.18.252:44961
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/krb5kdc/kudu_127.1.18.252.keytab
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.18.252
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.18.252:44961
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37717/jwks.json
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:23:37.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/info.pb
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.18.252
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32985
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/logs
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:40:09 UTC on ccycloud.quasar-gjkhih.root.comops.site
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457883
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:23:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.781064 1530 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:23:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.782048 1530 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:23:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.799556 1537 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.799643 1539 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:37.800056 1536 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:23:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:37.801713 1530 server_base.cc:1034] running on GCE node
05:23:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.029537 1530 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:23:39.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.031993 1530 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:23:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.033387 1530 hybrid_clock.cc:648] HybridClock initialized: now 1782019419033363 us; error 43 us; skew 500 ppm
05:23:39.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.034279 1530 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:23:39.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:39 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019419, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:39.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.040042 1530 init.cc:377] Logged in from keytab as kudu/127.1.18.252@KRBTEST.COM (short username kudu)
05:23:39.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.044490 1530 webserver.cc:460] Webserver started at http://127.1.18.252:32985/ using document root <none> and password file <none>
05:23:39.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.045578 1530 fs_manager.cc:359] Metadata directory not provided
05:23:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.045855 1530 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:23:39.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.053766 1530 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.003s
05:23:39.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.058049 1548 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:23:39.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.059079 1530 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.059489 1530 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data,/tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f1549ff959894c3288479c3861335599"
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:23:33 on dist-test-slave-n3h2"
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.062014 1530 fs_report.cc:383] FS layout report
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/wal
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/data
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:23:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:23:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:23:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:23:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:23:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:23:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:23:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:23:39.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:23:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.114831 1530 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:23:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.119908 1530 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:23:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.120481 1530 kserver.cc:163] Server-wide thread pool size limit: 3276
05:23:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.195936 1530 rpc_server.cc:307] RPC server started. Bound to: 127.1.18.252:44961
05:23:39.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.196105 1599 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.18.252:44961 every 8 connection(s)
05:23:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.198733 1530 server_base.cc:1166] Dumped server information to /tmp/dist-test-task55sErp/test-tmp/mini-kudu-cluster6605061949559097440/master-2/data/info.pb
05:23:39.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.201753 1099 external_mini_cluster.cc:1388] Started /tmp/dist-test-task55sErp/build/tsan/bin/kudu as pid 1530
05:23:39.202 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:23:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.209048 1600 sys_catalog.cc:263] Verifying existing consensus state
05:23:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.216137 1600 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Bootstrap starting.
05:23:39.250 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:39.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.264024 1600 log.cc:824] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Log is configured to *not* fsync() on all Append() calls
05:23:39.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.271261 1600 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:23:39.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.272114 1600 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Bootstrap complete.
05:23:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.307642 1600 raft_consensus.cc:348] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:39.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.310976 1600 raft_consensus.cc:374] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:23:39.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.311278 1600 raft_consensus.cc:729] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f1549ff959894c3288479c3861335599, State: Initialized, Role: FOLLOWER
05:23:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.312300 1600 consensus_queue.cc:260] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:39.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.315150 1604 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } } }
05:23:39.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.316087 1604 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [sys.catalog]: This master's current role is: FOLLOWER
05:23:39.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.316916 1600 sys_catalog.cc:564] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [sys.catalog]: configured and running, proceeding with master startup.
05:23:39.318 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:39.321 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.18.252:44961(127.1.18.252:44961)] connecting to peer
05:23:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:39.370472 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:39.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:39.370795 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:39.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:39.375882 1554 tablet.cc:2369] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:23:39.392 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.393 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.394 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.394 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.395 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.397 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.398 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.18.252:44961(127.1.18.252:44961)] Successfully connected to peer
05:23:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.364238 1530 master_runner.cc:186] Error getting master registration for 127.1.18.254:45727: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111),
05:23:39.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.403419 1530 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:39.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.470331 1604 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:23:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.470908 1604 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:39.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:39.476133 1549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:23:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.478163 1604 leader_election.cc:288] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727), 4d883a5df4994fda874847144e000593 (127.1.18.253:38429)
05:23:39.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:39.493433 1552 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4d883a5df4994fda874847144e000593 (127.1.18.253:38429): Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:39.494 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:23:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:39.501828 1549 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727): Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:23:39.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.502312 1549 leader_election.cc:302] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f1549ff959894c3288479c3861335599; no voters: 4d883a5df4994fda874847144e000593, c5eeec8ee53e4ddea4b534bad768bab6
05:23:39.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:39.503073 1604 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:23:39.506 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.18.252
05:23:39.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:39 dist-test-slave-n3h2 krb5kdc[1111](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019413, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.18.252@KRBTEST.COM
05:23:39.734 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x13e20325, L:/127.0.0.1:38554 - R:/127.1.18.252:44961] using SASL/GSSAPI
05:23:39.747 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:39.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:39.770 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:39.779 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.782 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.783 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.786 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:39.807 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:39.809 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:39.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.811 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.812 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.812 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.812 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.815 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:39.827 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:39.828 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:39.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.830 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.834 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.834 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.835 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.838 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:39.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:39.868 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:39.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.870 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.874 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.878 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.878 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.879 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:39.908 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:39.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:39.910 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.910 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.915 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:39.967 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:39.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:39.970 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.971 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:39.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:39.976 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:40.088 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:40.090 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:40.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.091 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.093 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.095 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:40.148 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:40.149 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:40.151 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.151 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.152 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.159 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.159 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.160 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.160 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:40.375029 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:40.375334 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:40.607 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:40.608 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:40.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.610 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.611 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.611 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.612 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:40.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:40.614 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:41.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:41.140583 1623 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:23:41.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:41.141290 1623 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:41.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:41.143847 1623 leader_election.cc:288] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727), 4d883a5df4994fda874847144e000593 (127.1.18.253:38429)
05:23:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:41.151881 1549 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727): Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:23:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:41.152383 1552 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4d883a5df4994fda874847144e000593 (127.1.18.253:38429): Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:41.152779 1552 leader_election.cc:302] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f1549ff959894c3288479c3861335599; no voters: 4d883a5df4994fda874847144e000593, c5eeec8ee53e4ddea4b534bad768bab6
05:23:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:41.153350 1623 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:23:41.188 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:41.189 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:41.190 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:41.190 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:41.190 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:41.191 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:41.192 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:41.192 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:41.194 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:41.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:41.379303 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:41.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:41.379549 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:42.382328 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:42.382627 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:42.827 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:42.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:42.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:42.830 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:42.830 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:42.833 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:42.833 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:42.833 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:42.835 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:43.210664 1630 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:23:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:43.210994 1630 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:43.212495 1630 leader_election.cc:288] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727), 4d883a5df4994fda874847144e000593 (127.1.18.253:38429)
05:23:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:43.216713 1549 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727): Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:23:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:43.217133 1552 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4d883a5df4994fda874847144e000593 (127.1.18.253:38429): Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:43.217453 1552 leader_election.cc:302] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f1549ff959894c3288479c3861335599; no voters: 4d883a5df4994fda874847144e000593, c5eeec8ee53e4ddea4b534bad768bab6
05:23:43.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:43.218008 1630 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:23:43.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:43.385810 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:43.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:43.386045 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:44.388628 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:44.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:44.388861 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:44.405861 1530 master_runner.cc:186] Error getting master registration for 127.1.18.254:45727: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111),
05:23:44.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:44.406191 1530 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:45.391680 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:45.391971 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:45.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:45.496263 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1000 after lost signal to thread 460
05:23:45.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:45.496970 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000a1400 after lost signal to thread 528
05:23:45.987 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:45.988 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:45.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:45.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:45.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:45.992 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:45.992 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:45.992 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:45.994 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:46.394682 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:46.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:46.394966 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:46.627 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:46.629 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:46.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:46.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:46.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:46.632 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:46.632 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:46.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:46.634 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:46.828 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:46.829 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:46.831 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:46.831 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:46.831 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:46.834 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:46.834 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:46.834 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:46.836 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:46.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:46.852711 1642 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:23:46.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:46.853042 1642 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:46.854676 1642 leader_election.cc:288] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727), 4d883a5df4994fda874847144e000593 (127.1.18.253:38429)
05:23:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:46.856667 1549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:23:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:46.859668 1549 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727): Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:23:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:46.860090 1552 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4d883a5df4994fda874847144e000593 (127.1.18.253:38429): Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:46.860548 1552 leader_election.cc:302] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f1549ff959894c3288479c3861335599; no voters: 4d883a5df4994fda874847144e000593, c5eeec8ee53e4ddea4b534bad768bab6
05:23:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:46.861304 1642 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:23:47.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:47.397789 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:47.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:47.398067 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:48.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:48.401582 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:48.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:48.401899 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:48.907 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:48.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:48.910 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:48.911 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:48.911 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:48.912 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:48.912 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:48.912 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:48.914 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:49 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019429, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:49.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:49.043771 1544 init.cc:295] Successfully reacquired a new kerberos TGT
05:23:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:49.404827 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:49.405061 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:49.409281 1530 master_runner.cc:186] Error getting master registration for 127.1.18.254:45727: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111),
05:23:49.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:49.409713 1530 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:50.407871 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:50.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:50.408159 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:51.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:51.410979 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:51.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:51.411221 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:52.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:52.304558 1653 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:23:52.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:52.304958 1653 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:23:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:52.306389 1653 leader_election.cc:288] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727), 4d883a5df4994fda874847144e000593 (127.1.18.253:38429)
05:23:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:52.308562 1549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:23:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:52.311046 1552 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4d883a5df4994fda874847144e000593 (127.1.18.253:38429): Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:23:52.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:52.311491 1549 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727): Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:23:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:52.311821 1549 leader_election.cc:302] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f1549ff959894c3288479c3861335599; no voters: 4d883a5df4994fda874847144e000593, c5eeec8ee53e4ddea4b534bad768bab6
05:23:52.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:52.312384 1653 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:23:52.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:52.413893 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:52.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:52.414121 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:52.627 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:52.628 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:52.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:52.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:52.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:52.630 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:52.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:52.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:52.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:53.417071 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:53.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:53.417320 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:54.411989 1530 master_runner.cc:186] Error getting master registration for 127.1.18.254:45727: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111),
05:23:54.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:54.412391 1530 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:54.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:54.420025 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:54.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:54.420387 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:54.547 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:54.549 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:54.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:54.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:54.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:54.552 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:54.552 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:54.552 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:54.554 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:55.423241 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:55.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:55.423470 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:56.426169 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:56.426398 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:56.668 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:56.669 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:56.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:56.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:56.671 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:56.672 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:56.672 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:56.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:56.675 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:56.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:23:56.928 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:23:56.930 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:56.930 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:56.930 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:23:56.931 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:56.931 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:56.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:23:56.933 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:23:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:57.429289 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:57.429526 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:58.432928 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:58.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:58.433295 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:23:59.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:23:59 dist-test-slave-n3h2 krb5kdc[1111](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019439, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.18.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:23:59.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:59.046146 1544 init.cc:295] Successfully reacquired a new kerberos TGT
05:23:59.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:59.415299 1530 master_runner.cc:186] Error getting master registration for 127.1.18.254:45727: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111),
05:23:59.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:23:59.415699 1530 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:23:59.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:59.435978 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:23:59.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:23:59.436280 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:00.439237 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:00.439558 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:00.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:24:00.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:24:00.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:00.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:00.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:00.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:00.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:00.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:00.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:24:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:01.442586 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:01.442915 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:02.445788 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:02.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:02.446045 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:03.120599 1668 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:24:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:03.121037 1668 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:24:03.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:03.122532 1668 leader_election.cc:288] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727), 4d883a5df4994fda874847144e000593 (127.1.18.253:38429)
05:24:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:03.124787 1549 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:24:03.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:03.128641 1549 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727): Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:24:03.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:03.129741 1552 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4d883a5df4994fda874847144e000593 (127.1.18.253:38429): Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:24:03.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:03.130110 1552 leader_election.cc:302] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f1549ff959894c3288479c3861335599; no voters: 4d883a5df4994fda874847144e000593, c5eeec8ee53e4ddea4b534bad768bab6
05:24:03.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:03.130771 1668 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:24:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:03.448933 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:03.449177 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:03.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:24:03.928 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:24:03.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:03.928 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:03.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:03.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:03.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:03.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:03.932 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:24:04.407 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:24:04.407 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:24:04.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:04.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:04.408 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:04.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:04.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:04.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:04.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:24:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:04.417956 1530 master_runner.cc:186] Error getting master registration for 127.1.18.254:45727: Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111),
05:24:04.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:04.418351 1530 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:24:04.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:24:04.447 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:24:04.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:04.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:04.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:04.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:04.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:04.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:04.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:24:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:04.453068 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:04.453384 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:05.456099 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:05.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:05.456363 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:06.458940 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:06.459182 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:06.480530 1677 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:24:06.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:06.480952 1677 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c5eeec8ee53e4ddea4b534bad768bab6" member_type: VOTER last_known_addr { host: "127.1.18.254" port: 45727 } } peers { permanent_uuid: "4d883a5df4994fda874847144e000593" member_type: VOTER last_known_addr { host: "127.1.18.253" port: 38429 } } peers { permanent_uuid: "f1549ff959894c3288479c3861335599" member_type: VOTER last_known_addr { host: "127.1.18.252" port: 44961 } }
05:24:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:06.482731 1677 leader_election.cc:288] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727), 4d883a5df4994fda874847144e000593 (127.1.18.253:38429)
05:24:06.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:06.488818 1549 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c5eeec8ee53e4ddea4b534bad768bab6 (127.1.18.254:45727): Network error: Client connection negotiation failed: client connection to 127.1.18.254:45727: connect: Connection refused (error 111)
05:24:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:06.489791 1552 leader_election.cc:334] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4d883a5df4994fda874847144e000593 (127.1.18.253:38429): Network error: Client connection negotiation failed: client connection to 127.1.18.253:38429: connect: Connection refused (error 111)
05:24:06.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:06.490121 1552 leader_election.cc:302] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: f1549ff959894c3288479c3861335599; no voters: 4d883a5df4994fda874847144e000593, c5eeec8ee53e4ddea4b534bad768bab6
05:24:06.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:06.490693 1677 raft_consensus.cc:2726] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:24:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:07.461822 1617 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f1549ff959894c3288479c3861335599: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:24:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:24:07.462066 1617 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:24:08.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.253:38429(127.1.18.253:38429)] connecting to peer
05:24:08.048 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.18.254:45727(127.1.18.254:45727)] connecting to peer
05:24:08.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:08.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.18.253:38429(127.1.18.253:38429)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:08.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.253:38429: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429
05:24:08.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:08.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.254:45727(127.1.18.254:45727)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:08.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.18.254:45727: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
05:24:08.053 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961; will retry
05:24:08.057 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28848), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [535ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [538ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [572ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [574ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [576ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [600ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [669ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [700ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [877ms] refreshing cache from master, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [881ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [941ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [949ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [950ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1977ms] refreshing cache from master, [1977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1982ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1983ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1984ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3617ms] refreshing cache from master, [3617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [3620ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [3621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [3623ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3625ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [3625ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [6777ms] refreshing cache from master, [6777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [6778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [6779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [6780ms] too many traces: truncated at 100 traces, deferred=Deferred@296698757(state=PENDING, result=null, callback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339), errback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339)))
05:24:08.062 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.253:38429(127.1.18.253:38429)] disconnecting while in state TERMINATED
05:24:08.063 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.252:44961(127.1.18.252:44961)] disconnecting while in state READY
05:24:08.063 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.18.254:45727(127.1.18.254:45727)] disconnecting while in state TERMINATED
05:24:08.068 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:24:08.069 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:24:08.070 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.18.252:44961(127.1.18.252:44961)] handling channelInactive
05:24:08.071 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.18.252:44961(127.1.18.252:44961)] cleaning up while in state READY due to: connection closed
05:24:08.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:08.077175 1099 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 1530
05:24:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:24:09Z chronyd exiting
05:24:10.291 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:24:10.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:10.292256 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 653
05:24:10.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:10.317407 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 810
05:24:10.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:10.342994 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 957
05:24:10.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:10.374274 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 386
05:24:10.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:10.402766 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 456
05:24:10.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:24:10.431164 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task55sErp/build/tsan/bin/kudu with pid 531
05:24:11.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:24:11Z chronyd exiting
05:24:12.599 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28848), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [535ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [538ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [572ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [574ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [576ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [600ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [669ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [700ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [877ms] refreshing cache from master, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [881ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [941ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [949ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [950ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1977ms] refreshing cache from master, [1977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1982ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1983ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1984ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3617ms] refreshing cache from master, [3617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [3620ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [3621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [3623ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3625ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [3625ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [6777ms] refreshing cache from master, [6777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [6778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [6779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [6780ms] too many traces: truncated at 100 traces, deferred=Deferred@296698757(state=PENDING, result=null, callback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339), errback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.254:45727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28848), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [535ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [538ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [572ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [574ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [576ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [600ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [669ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [700ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [877ms] refreshing cache from master, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [881ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [941ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [949ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [950ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1977ms] refreshing cache from master, [1977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1982ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1983ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1984ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3617ms] refreshing cache from master, [3617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [3620ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [3621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [3623ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3625ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [3625ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [6777ms] refreshing cache from master, [6777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [6778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [6779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [6780ms] too many traces: truncated at 100 traces, deferred=Deferred@296698757(state=PENDING, result=null, callback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339), errback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.254:45727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
05:24:13.022 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
05:24:13.022 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28848), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [535ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [538ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [572ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [574ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [576ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [600ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [669ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [700ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [877ms] refreshing cache from master, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [881ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [941ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [949ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [950ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1977ms] refreshing cache from master, [1977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1982ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1983ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1984ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3617ms] refreshing cache from master, [3617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [3620ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [3621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [3623ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3625ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [3625ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [6777ms] refreshing cache from master, [6777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [6778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [6779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [6780ms] too many traces: truncated at 100 traces, deferred=Deferred@296698757(state=PENDING, result=null, callback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339), errback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.254:45727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=28848), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [108ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [184ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [535ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [538ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [560ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [561ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [572ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [574ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [576ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [597ms] refreshing cache from master, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [600ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [604ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [605ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [619ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [627ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [628ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [656ms] refreshing cache from master, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [660ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [660ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [667ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [669ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [669ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [700ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [765ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [766ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [877ms] refreshing cache from master, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [881ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [883ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [885ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [937ms] refreshing cache from master, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [939ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [941ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [949ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [950ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1396ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1399ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1977ms] refreshing cache from master, [1977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [1979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [1980ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [1982ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [1983ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [1984ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3617ms] refreshing cache from master, [3617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [3618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [3620ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.253:38429: Network error: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429, [3621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [3623ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.254:45727: Network error: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [3625ms] Sub RPC ConnectToMaster: received response from server master-127.1.18.252:44961: OK, [3625ms] delaying RPC due to: Service unavailable: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727, [6777ms] refreshing cache from master, [6777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.253:38429, [6778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.254:45727, [6779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.18.252:44961, [6780ms] too many traces: truncated at 100 traces, deferred=Deferred@296698757(state=PENDING, result=null, callback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339), errback=(continuation of Deferred@505536193 after retry RPC after error@1405071283) -> (continuation of Deferred@665788173 after retry RPC after error@1282917142) -> (continuation of Deferred@1546701711 after retry RPC after error@1918641549) -> (continuation of Deferred@760482748 after retry RPC after error@1607012979) -> (continuation of Deferred@100652726 after retry RPC after error@370998217) -> (continuation of Deferred@462953309 after retry RPC after error@206952663) -> (continuation of Deferred@2003105165 after retry RPC after error@106400283) -> (continuation of Deferred@1565404031 after retry RPC after error@2139079521) -> (continuation of Deferred@803707294 after retry RPC after error@366659883) -> (continuation of Deferred@962792764 after retry RPC after error@948352442) -> (continuation of Deferred@733933661 after retry RPC after error@1760491408) -> (continuation of Deferred@1229262172 after retry RPC after error@1147357867) -> (continuation of Deferred@411264610 after retry RPC after error@122572949) -> (continuation of Deferred@1711313613 after retry RPC after error@946387636) -> (continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.18.253:38429,127.1.18.254:45727,127.1.18.252:44961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.253:38429(127.1.18.253:38429): Connection refused: /127.1.18.253:38429,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.18.254:45727(127.1.18.254:45727): Connection refused: /127.1.18.254:45727
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.18.254:45727
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)