Unable to diagnose

21:45:10.726 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:45:10.738 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskAO3oum/test-tmp
21:45:10.741 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:45:10.745 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskAO3oum/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:45:11.038 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:45:11.178 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:45:11.179 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:45:11.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.184967 30992 mini_oidc.cc:152] Starting JWKS server
21:45:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.189769 30992 webserver.cc:460] Webserver started at http://127.0.0.1:37423/ using document root <none> and password file <none>
21:45:11.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.190253 30992 mini_oidc.cc:189] Starting OIDC Discovery server
21:45:11.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.192605 30992 webserver.cc:460] Webserver started at http://127.0.0.1:35373/ using document root <none> and password file <none>
21:45:11.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:45:11Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
21:45:11.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:45:11Z Disabled control of system clock
21:45:11.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.263996 30992 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/wal
21:45:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data
21:45:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/logs
21:45:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data/info.pb
21:45:11.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:11.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.62:40071
21:45:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.62
21:45:11.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:11.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:11.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:11.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:11.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37423/jwks.json with env {}
21:45:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.585369 31004 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.586017 31004 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.586841 31004 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.625936 31004 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:11.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.626261 31004 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.626466 31004 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.626657 31004 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.626929 31004 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.627101 31004 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37423/jwks.json
21:45:11.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.663106 31004 master_runner.cc:386] Master server non-default flags:
21:45:11.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:11.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data
21:45:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/wal
21:45:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.62:40071
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37423/jwks.json
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data/info.pb
21:45:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.62
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/logs
21:45:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.664530 31004 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.666343 31004 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:11.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.683451 31012 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.683794 31011 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:11.684381 31014 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:11.685595 31004 server_base.cc:1034] running on GCE node
21:45:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.817622 31004 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.820724 31004 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:12.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.822160 31004 hybrid_clock.cc:648] HybridClock initialized: now 1714686312822107 us; error 55 us; skew 500 ppm
21:45:12.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.823166 31004 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:12.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.830186 31004 webserver.cc:460] Webserver started at http://127.30.68.62:44547/ using document root <none> and password file <none>
21:45:12.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.831369 31004 fs_manager.cc:359] Metadata directory not provided
21:45:12.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.831619 31004 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:12.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.832228 31004 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.837742 31004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data/instance:
21:45:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "404a366dc2b24c638df917e38d7dff18"
21:45:12.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:12 on dist-test-slave-zqx8"
21:45:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.839263 31004 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/wal/instance:
21:45:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "404a366dc2b24c638df917e38d7dff18"
21:45:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:12 on dist-test-slave-zqx8"
21:45:12.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.847345 31004 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
21:45:12.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.853605 31021 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:12.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.854795 31004 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
21:45:12.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.855193 31004 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/wal
21:45:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "404a366dc2b24c638df917e38d7dff18"
21:45:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:12 on dist-test-slave-zqx8"
21:45:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.855657 31004 fs_report.cc:383] FS layout report
21:45:12.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/wal
21:45:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/wal
21:45:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data/data
21:45:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:12.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:12.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:12.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.918418 31004 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.920162 31004 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:12.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.920775 31004 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.997563 31004 rpc_server.cc:307] RPC server started. Bound to: 127.30.68.62:40071
21:45:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:12.997684 31073 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.68.62:40071 every 8 connection(s)
21:45:13.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.000663 31004 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/data/info.pb
21:45:13.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.006596 31074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:45:13.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.006974 30992 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31004
21:45:13.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.007522 30992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-0/wal/instance
21:45:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.013028 30992 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/wal
21:45:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data
21:45:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:13.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/logs
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data/info.pb
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:13.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.61:37547
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.61
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:13.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:13.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37423/jwks.json with env {}
21:45:13.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.023563 31074 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } has no permanent_uuid. Determining permanent_uuid...
21:45:13.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.050794 31074 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } has no permanent_uuid. Determining permanent_uuid...
21:45:13.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.053532 31023 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:13.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.057888 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.61:37547: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:13.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.107452 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } attempt: 1
21:45:13.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.111810 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.61:37547: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:13.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.180231 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } attempt: 2
21:45:13.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.184839 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.61:37547: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:13.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.276243 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } attempt: 3
21:45:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.280316 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.61:37547: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.347165 31076 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.347767 31076 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.348426 31076 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:13.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.380983 31076 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:13.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.381296 31076 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:13.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.381503 31076 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:13.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.381690 31076 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.381891 31076 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:13.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.382079 31076 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37423/jwks.json
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.418676 31076 master_runner.cc:386] Master server non-default flags:
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/wal
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:13.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.61:37547
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37423/jwks.json
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data/info.pb
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.61
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/logs
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.420145 31076 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:13.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.422776 31076 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.423791 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } attempt: 4
21:45:13.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.427794 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.61:37547: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:13.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.439709 31086 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:13.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.727309 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } attempt: 5
21:45:13.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.731438 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.61:37547: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.279187 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } attempt: 6
21:45:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:14.290081 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.61:37547: Network error: Client connection negotiation failed: client connection to 127.30.68.61:37547: connect: Connection refused (error 111)
21:45:14.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.442261 31088 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:13.443471 31076 server_base.cc:1034] running on GCE node
21:45:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:13.440963 31085 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.561723 31076 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:14.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.564563 31076 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:14.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.565976 31076 hybrid_clock.cc:648] HybridClock initialized: now 1714686314565942 us; error 49 us; skew 500 ppm
21:45:14.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.567080 31076 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.574561 31076 webserver.cc:460] Webserver started at http://127.30.68.61:46675/ using document root <none> and password file <none>
21:45:14.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.575577 31076 fs_manager.cc:359] Metadata directory not provided
21:45:14.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.575780 31076 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:14.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.576278 31076 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.581558 31076 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data/instance:
21:45:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6350a306f48046d7b246822c2787d4e6"
21:45:14.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:14 on dist-test-slave-zqx8"
21:45:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.582712 31076 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/wal/instance:
21:45:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6350a306f48046d7b246822c2787d4e6"
21:45:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:14 on dist-test-slave-zqx8"
21:45:14.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.590217 31076 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.007s
21:45:14.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.596136 31097 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:14.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.597368 31076 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
21:45:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.597754 31076 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/wal
21:45:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6350a306f48046d7b246822c2787d4e6"
21:45:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:14 on dist-test-slave-zqx8"
21:45:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.598166 31076 fs_report.cc:383] FS layout report
21:45:14.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:14.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/wal
21:45:14.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/wal
21:45:14.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data/data
21:45:14.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:14.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:14.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.652873 31076 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:14.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.654554 31076 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.655115 31076 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.727906 31076 rpc_server.cc:307] RPC server started. Bound to: 127.30.68.61:37547
21:45:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.728008 31149 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.68.61:37547 every 8 connection(s)
21:45:14.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.730906 31076 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/data/info.pb
21:45:14.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.733076 30992 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31076
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.733647 30992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-1/wal/instance
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.736320 30992 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/wal
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/logs
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data/info.pb
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:14.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.60:46113
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.60
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37423/jwks.json with env {}
21:45:14.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.737169 31150 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:45:14.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.750166 31150 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } has no permanent_uuid. Determining permanent_uuid...
21:45:14.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.775436 31150 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } has no permanent_uuid. Determining permanent_uuid...
21:45:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.790000 31150 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } has no permanent_uuid. Determining permanent_uuid...
21:45:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:14.792594 31098 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:14.797084 31150 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:14.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.846760 31150 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 1
21:45:14.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:14.851904 31150 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:14.920491 31150 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 2
21:45:14.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:14.925487 31150 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.017014 31150 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 3
21:45:15.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.021327 31150 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.069257 31152 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.069849 31152 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.070521 31152 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.102926 31152 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.103245 31152 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.103444 31152 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.103724 31152 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.104128 31152 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.104430 31152 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37423/jwks.json
21:45:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.141093 31152 master_runner.cc:386] Master server non-default flags:
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/wal
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.60:46113
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37423/jwks.json
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data/info.pb
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.60
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/logs
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:15.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.142453 31152 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:15.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.144392 31152 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:15.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.163440 31165 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.164810 31150 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 4
21:45:15.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.168633 31150 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.350836 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } attempt: 7
21:45:15.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.364037 31074 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } has no permanent_uuid. Determining permanent_uuid...
21:45:15.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.372397 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.430986 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 1
21:45:15.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.436499 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.468186 31150 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 5
21:45:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.472795 31150 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.518265 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 2
21:45:15.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.523409 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.609160 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 3
21:45:15.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.614171 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:15.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.754889 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 4
21:45:15.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.761299 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:16.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.020553 31150 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 6
21:45:16.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.028513 31150 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:16.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.044741 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 5
21:45:16.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.049643 31074 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.68.60:46113: Network error: Client connection negotiation failed: client connection to 127.30.68.60:46113: connect: Connection refused (error 111)
21:45:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.163558 31162 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:15.164072 31163 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:16.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:15.165184 31152 server_base.cc:1034] running on GCE node
21:45:16.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.299870 31152 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:16.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.302700 31152 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:16.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.304124 31152 hybrid_clock.cc:648] HybridClock initialized: now 1714686316304063 us; error 91 us; skew 500 ppm
21:45:16.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.305119 31152 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:16.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.311996 31152 webserver.cc:460] Webserver started at http://127.30.68.60:45005/ using document root <none> and password file <none>
21:45:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.313118 31152 fs_manager.cc:359] Metadata directory not provided
21:45:16.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.313364 31152 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:16.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.313915 31152 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:16.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.319299 31152 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data/instance:
21:45:16.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec90a921418f457487d0a8c84f862684"
21:45:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:16 on dist-test-slave-zqx8"
21:45:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.320820 31152 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/wal/instance:
21:45:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec90a921418f457487d0a8c84f862684"
21:45:16.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:16 on dist-test-slave-zqx8"
21:45:16.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.329228 31152 fs_manager.cc:691] Time spent creating directory manager: real 0.008s	user 0.006s	sys 0.001s
21:45:16.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.335760 31178 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.336933 31152 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
21:45:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.337272 31152 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/wal
21:45:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ec90a921418f457487d0a8c84f862684"
21:45:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:16 on dist-test-slave-zqx8"
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.337613 31152 fs_report.cc:383] FS layout report
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/wal
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/wal
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data/data
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:16.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:16.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.395126 31152 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:16.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.396780 31152 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:16.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.397282 31152 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:16.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.475425 31152 rpc_server.cc:307] RPC server started. Bound to: 127.30.68.60:46113
21:45:16.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.475566 31230 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.68.60:46113 every 8 connection(s)
21:45:16.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.478430 31152 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/data/info.pb
21:45:16.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.484014 31231 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:45:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.486055 30992 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31152
21:45:16.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.486598 30992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/master-2/wal/instance
21:45:16.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.503093 31231 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } has no permanent_uuid. Determining permanent_uuid...
21:45:16.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.541913 31231 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } has no permanent_uuid. Determining permanent_uuid...
21:45:16.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.552628 31231 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } has no permanent_uuid. Determining permanent_uuid...
21:45:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.569492 31231 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684: Bootstrap starting.
21:45:16.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.576225 31231 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684: Neither blocks nor log segments found. Creating new log.
21:45:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.578228 31231 log.cc:824] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684: Log is configured to *not* fsync() on all Append() calls
21:45:16.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.582731 31231 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684: No bootstrap required, opened a new log
21:45:16.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.602067 31074 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 6
21:45:16.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.607429 31231 raft_consensus.cc:348] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:16.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.610061 31231 raft_consensus.cc:374] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:45:16.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.610750 31231 raft_consensus.cc:729] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ec90a921418f457487d0a8c84f862684, State: Initialized, Role: FOLLOWER
21:45:16.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.611812 31231 consensus_queue.cc:260] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:16.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.615168 31241 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:16.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.616405 31241 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: This master's current role is: FOLLOWER
21:45:16.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.617267 31231 sys_catalog.cc:564] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: configured and running, proceeding with master startup.
21:45:16.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.628741 31074 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: Bootstrap starting.
21:45:16.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.637141 31074 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: Neither blocks nor log segments found. Creating new log.
21:45:16.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.642474 31074 log.cc:824] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: Log is configured to *not* fsync() on all Append() calls
21:45:16.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.649560 31074 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: No bootstrap required, opened a new log
21:45:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.651026 31252 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:16.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.651697 31252 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:16.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.652850 31184 tablet.cc:2369] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:45:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.682955 31074 raft_consensus.cc:348] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.683904 31074 raft_consensus.cc:374] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:45:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.684206 31074 raft_consensus.cc:729] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 404a366dc2b24c638df917e38d7dff18, State: Initialized, Role: FOLLOWER
21:45:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.685691 31074 consensus_queue.cc:260] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:16.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.646939 31152 master_runner.cc:186] Error getting master registration for 127.30.68.62:40071: OK, instance_id { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" instance_seqno: 1714686312976052 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:45:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.686909 31152 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.689221 31255 sys_catalog.cc:455] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:16.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.690184 31255 sys_catalog.cc:458] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [sys.catalog]: This master's current role is: FOLLOWER
21:45:16.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.691148 31074 sys_catalog.cc:564] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [sys.catalog]: configured and running, proceeding with master startup.
21:45:16.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.726904 31266 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:16.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.727720 31266 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.727319 31004 master_runner.cc:186] Error getting master registration for 127.30.68.61:37547: OK, instance_id { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" instance_seqno: 1714686314708207 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:45:16.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:16.771119 31004 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:16.930835 31027 tablet.cc:2369] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:45:17.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.023804 31241 raft_consensus.cc:482] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:45:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.024425 31241 raft_consensus.cc:504] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.031663 31049 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ec90a921418f457487d0a8c84f862684" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "404a366dc2b24c638df917e38d7dff18" is_pre_election: true
21:45:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.033217 31049 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ec90a921418f457487d0a8c84f862684 in term 0.
21:45:17.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.035661 31181 leader_election.cc:302] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 404a366dc2b24c638df917e38d7dff18, ec90a921418f457487d0a8c84f862684; no voters: 
21:45:17.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.038200 31241 leader_election.cc:288] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 404a366dc2b24c638df917e38d7dff18 (127.30.68.62:40071), 6350a306f48046d7b246822c2787d4e6 (127.30.68.61:37547)
21:45:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.039549 31241 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:45:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.040418 31241 raft_consensus.cc:482] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:45:17.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.041283 31241 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 0 FOLLOWER]: Advancing to term 1
21:45:17.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.052429 31241 raft_consensus.cc:504] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.055999 31049 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ec90a921418f457487d0a8c84f862684" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "404a366dc2b24c638df917e38d7dff18"
21:45:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.056697 31049 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [term 0 FOLLOWER]: Advancing to term 1
21:45:17.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.058578 31241 leader_election.cc:288] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [CANDIDATE]: Term 1 election: Requested vote from peers 404a366dc2b24c638df917e38d7dff18 (127.30.68.62:40071), 6350a306f48046d7b246822c2787d4e6 (127.30.68.61:37547)
21:45:17.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.067235 31049 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ec90a921418f457487d0a8c84f862684 in term 1.
21:45:17.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.069437 31181 leader_election.cc:302] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 404a366dc2b24c638df917e38d7dff18, ec90a921418f457487d0a8c84f862684; no voters: 
21:45:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.070386 31241 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 1 FOLLOWER]: Leader election won for term 1
21:45:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.070868 31241 raft_consensus.cc:686] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [term 1 LEADER]: Becoming Leader. State: Replica: ec90a921418f457487d0a8c84f862684, State: Running, Role: LEADER
21:45:17.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.071861 31241 consensus_queue.cc:237] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.083355 31268 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ec90a921418f457487d0a8c84f862684. Latest consensus state: current_term: 1 leader_uuid: "ec90a921418f457487d0a8c84f862684" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.084218 31268 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: This master's current role is: LEADER
21:45:17.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.089154 31150 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } attempt: 7
21:45:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.091117 31270 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
21:45:17.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.099028 31270 catalog_manager.cc:1484] Initializing Kudu cluster ID...
21:45:17.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.128845 31049 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [term 1 FOLLOWER]: Refusing update from remote peer ec90a921418f457487d0a8c84f862684: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:45:17.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.131908 31268 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [LEADER]: Connected to new peer: Peer: permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:45:17.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.157001 31150 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6: Bootstrap starting.
21:45:17.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.165122 31150 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6: Neither blocks nor log segments found. Creating new log.
21:45:17.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.168347 31150 log.cc:824] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6: Log is configured to *not* fsync() on all Append() calls
21:45:17.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.187175 31150 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6: No bootstrap required, opened a new log
21:45:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.189433 31255 sys_catalog.cc:455] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ec90a921418f457487d0a8c84f862684. Latest consensus state: current_term: 1 leader_uuid: "ec90a921418f457487d0a8c84f862684" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.190270 31255 sys_catalog.cc:458] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [sys.catalog]: This master's current role is: FOLLOWER
21:45:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.203868 31272 mvcc.cc:205] Tried to move back new op lower bound from 7023355154931245056 to 7023355154738626560. Current Snapshot: MvccSnapshot[applied={T|T < 7023355154931245056}]
21:45:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.208750 31273 mvcc.cc:205] Tried to move back new op lower bound from 7023355154931245056 to 7023355154738626560. Current Snapshot: MvccSnapshot[applied={T|T < 7023355154931245056}]
21:45:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.217125 31241 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ec90a921418f457487d0a8c84f862684" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.217983 31241 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: This master's current role is: LEADER
21:45:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.219940 31270 catalog_manager.cc:1347] Generated new cluster ID: e5541eeb2fe04151a53ef819e4cb43ab
21:45:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.220263 31270 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
21:45:17.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.220749 31255 sys_catalog.cc:455] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ec90a921418f457487d0a8c84f862684" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.221532 31255 sys_catalog.cc:458] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18 [sys.catalog]: This master's current role is: FOLLOWER
21:45:17.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.253383 31270 catalog_manager.cc:1370] Generated new certificate authority record
21:45:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.262919 31270 catalog_manager.cc:1504] Loading token signing keys...
21:45:17.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.263509 31150 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:17.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.265867 31150 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:45:17.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.266672 31150 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6350a306f48046d7b246822c2787d4e6, State: Initialized, Role: FOLLOWER
21:45:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.269170 31150 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } }
21:45:17.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.281555 31278 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.282485 31278 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [sys.catalog]: This master's current role is: FOLLOWER
21:45:17.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.287050 31123 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ec90a921418f457487d0a8c84f862684" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6350a306f48046d7b246822c2787d4e6"
21:45:17.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.287933 31123 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [term 0 FOLLOWER]: Advancing to term 1
21:45:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.292717 31124 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ec90a921418f457487d0a8c84f862684" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6350a306f48046d7b246822c2787d4e6" is_pre_election: true
21:45:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.297339 31123 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ec90a921418f457487d0a8c84f862684 in term 1.
21:45:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.298964 31122 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [term 1 FOLLOWER]: Refusing update from remote peer ec90a921418f457487d0a8c84f862684: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:45:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.307842 31241 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:45:17.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.308910 31150 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [sys.catalog]: configured and running, proceeding with master startup.
21:45:17.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.321278 31270 catalog_manager.cc:5857] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684: Generated new TSK 0
21:45:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.322893 31270 catalog_manager.cc:1514] Initializing in-progress tserver states...
21:45:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.356391 30992 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/wal
21:45:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data
21:45:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/logs
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data/info.pb
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.1:0
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.68.1
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.1
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:17.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:17.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:17.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:17.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:45:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.394332 31278 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ec90a921418f457487d0a8c84f862684. Latest consensus state: current_term: 1 leader_uuid: "ec90a921418f457487d0a8c84f862684" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.395184 31278 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [sys.catalog]: This master's current role is: FOLLOWER
21:45:17.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.394548 31284 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ec90a921418f457487d0a8c84f862684" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.397020 31284 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6 [sys.catalog]: This master's current role is: FOLLOWER
21:45:17.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.437439 31296 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:45:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.733757 31266 catalog_manager.cc:1259] Loaded cluster ID: e5541eeb2fe04151a53ef819e4cb43ab
21:45:17.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.734194 31266 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: loading cluster ID for follower catalog manager: success
21:45:17.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.741819 31266 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: acquiring CA information for follower catalog manager: success
21:45:17.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.763096 31266 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 404a366dc2b24c638df917e38d7dff18: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:45:17.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:17.844276 31290 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:17.844918 31290 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:17.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:17.845568 31290 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:17.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:17.879267 31290 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:17.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:17.880313 31290 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.68.1
21:45:17.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.908385 31241 sys_catalog.cc:455] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ec90a921418f457487d0a8c84f862684" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "404a366dc2b24c638df917e38d7dff18" member_type: VOTER last_known_addr { host: "127.30.68.62" port: 40071 } } peers { permanent_uuid: "6350a306f48046d7b246822c2787d4e6" member_type: VOTER last_known_addr { host: "127.30.68.61" port: 37547 } } peers { permanent_uuid: "ec90a921418f457487d0a8c84f862684" member_type: VOTER last_known_addr { host: "127.30.68.60" port: 46113 } } }
21:45:17.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.909636 31241 sys_catalog.cc:458] T 00000000000000000000000000000000 P ec90a921418f457487d0a8c84f862684 [sys.catalog]: This master's current role is: LEADER
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.936401 31290 tablet_server_runner.cc:78] Tablet server non-default flags:
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/wal
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:17.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.1:0
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data/info.pb
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.1
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.68.1
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/logs
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:17.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:17.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:17.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.938170 31290 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:17.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:17.940312 31290 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:17.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:17.962577 31305 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:18.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:18.466246 31296 catalog_manager.cc:1259] Loaded cluster ID: e5541eeb2fe04151a53ef819e4cb43ab
21:45:18.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:18.470916 31296 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6: loading cluster ID for follower catalog manager: success
21:45:18.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:18.545626 31296 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6: acquiring CA information for follower catalog manager: success
21:45:18.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:18.585628 31296 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6350a306f48046d7b246822c2787d4e6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:45:19.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:19.356528 31303 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 31290
21:45:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:19.879877 31290 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.920s	user 0.516s	sys 0.784s
21:45:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:17.964255 31304 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:19.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:19.880316 31290 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.921s	user 0.516s	sys 0.784s
21:45:19.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:19.889935 31312 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:19.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.891016 31290 server_base.cc:1034] running on GCE node
21:45:19.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.892397 31290 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:19.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.895198 31290 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:19.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.896585 31290 hybrid_clock.cc:648] HybridClock initialized: now 1714686319896527 us; error 53 us; skew 500 ppm
21:45:19.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.897718 31290 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:19.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.905858 31290 webserver.cc:460] Webserver started at http://127.30.68.1:43923/ using document root <none> and password file <none>
21:45:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.907145 31290 fs_manager.cc:359] Metadata directory not provided
21:45:19.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.907444 31290 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:19.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.908025 31290 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:19.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.915642 31290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data/instance:
21:45:19.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33cc3160d76647c99191d4f964adaa86"
21:45:19.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:19 on dist-test-slave-zqx8"
21:45:19.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.917327 31290 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/wal/instance:
21:45:19.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33cc3160d76647c99191d4f964adaa86"
21:45:19.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:19 on dist-test-slave-zqx8"
21:45:19.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.927028 31290 fs_manager.cc:691] Time spent creating directory manager: real 0.009s	user 0.006s	sys 0.003s
21:45:19.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.934806 31318 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:19.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.936235 31290 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.004s	sys 0.002s
21:45:19.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.936703 31290 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/wal
21:45:19.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33cc3160d76647c99191d4f964adaa86"
21:45:19.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:19 on dist-test-slave-zqx8"
21:45:19.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:19.937186 31290 fs_report.cc:383] FS layout report
21:45:19.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/wal
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/wal
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data/data
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:19.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:20.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.002300 31290 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.004335 31290 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.005040 31290 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.008647 31290 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
21:45:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.014881 31290 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:45:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.015174 31290 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:45:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.015480 31290 ts_tablet_manager.cc:590] Registered 0 tablets
21:45:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.015733 31290 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:45:20.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.178414 31290 rpc_server.cc:307] RPC server started. Bound to: 127.30.68.1:41599
21:45:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.178480 31432 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.68.1:41599 every 8 connection(s)
21:45:20.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.182348 31290 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/data/info.pb
21:45:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.192497 30992 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31290
21:45:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.193171 30992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-0/wal/instance
21:45:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.210918 30992 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/wal
21:45:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data
21:45:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/logs
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data/info.pb
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.2:0
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.68.2
21:45:20.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.2
21:45:20.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:20.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:20.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:20.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:20.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:20.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:45:20.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.234194 31433 heartbeater.cc:347] Connected to a master server at 127.30.68.60:46113
21:45:20.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.235663 31433 heartbeater.cc:464] Registering TS with master...
21:45:20.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.237169 31433 heartbeater.cc:511] Master 127.30.68.60:46113 requested a full tablet report, sending...
21:45:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.240593 31196 ts_manager.cc:194] Registered new tserver with Master: 33cc3160d76647c99191d4f964adaa86 (127.30.68.1:41599)
21:45:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.241561 31434 heartbeater.cc:347] Connected to a master server at 127.30.68.61:37547
21:45:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.241923 31434 heartbeater.cc:464] Registering TS with master...
21:45:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.242717 31434 heartbeater.cc:511] Master 127.30.68.61:37547 requested a full tablet report, sending...
21:45:20.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.243324 31435 heartbeater.cc:347] Connected to a master server at 127.30.68.62:40071
21:45:20.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.243304 31196 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.68.1:49477
21:45:20.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.243667 31435 heartbeater.cc:464] Registering TS with master...
21:45:20.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.244642 31435 heartbeater.cc:511] Master 127.30.68.62:40071 requested a full tablet report, sending...
21:45:20.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.246094 31114 ts_manager.cc:194] Registered new tserver with Master: 33cc3160d76647c99191d4f964adaa86 (127.30.68.1:41599)
21:45:20.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.247669 31037 ts_manager.cc:194] Registered new tserver with Master: 33cc3160d76647c99191d4f964adaa86 (127.30.68.1:41599)
21:45:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.583174 31443 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.583776 31443 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.584385 31443 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:20.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.617691 31443 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:20.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.618881 31443 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.68.2
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.654660 31443 tablet_server_runner.cc:78] Tablet server non-default flags:
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/wal
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.2:0
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data/info.pb
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.2
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.68.2
21:45:20.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/logs
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:20.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:20.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:20.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.656246 31443 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:20.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.658501 31443 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:20.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.678792 31452 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.248778 31433 heartbeater.cc:503] Master 127.30.68.60:46113 was elected leader, sending a full tablet report...
21:45:21.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.679049 31449 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:20.679602 31450 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:20.680720 31443 server_base.cc:1034] running on GCE node
21:45:21.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.873994 31443 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:21.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.876788 31443 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:21.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.878212 31443 hybrid_clock.cc:648] HybridClock initialized: now 1714686321878181 us; error 48 us; skew 500 ppm
21:45:21.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.879128 31443 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:21.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.885958 31443 webserver.cc:460] Webserver started at http://127.30.68.2:39839/ using document root <none> and password file <none>
21:45:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.887032 31443 fs_manager.cc:359] Metadata directory not provided
21:45:21.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.887282 31443 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:21.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.887851 31443 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.893878 31443 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data/instance:
21:45:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6306970dc9a4c398c6958d62eb73b2d"
21:45:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:21 on dist-test-slave-zqx8"
21:45:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.895180 31443 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/wal/instance:
21:45:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6306970dc9a4c398c6958d62eb73b2d"
21:45:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:21 on dist-test-slave-zqx8"
21:45:21.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.903108 31443 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
21:45:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.909441 31461 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.910706 31443 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
21:45:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.911108 31443 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/wal
21:45:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6306970dc9a4c398c6958d62eb73b2d"
21:45:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:21 on dist-test-slave-zqx8"
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.911520 31443 fs_report.cc:383] FS layout report
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/wal
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/wal
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data/data
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:21.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:21.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:21.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.972295 31443 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.974089 31443 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:21.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.974596 31443 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:21.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.977253 31443 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
21:45:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.982385 31443 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:45:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.982826 31443 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.001s	sys 0.000s
21:45:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.983157 31443 ts_tablet_manager.cc:590] Registered 0 tablets
21:45:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:21.983338 31443 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:45:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.139952 31443 rpc_server.cc:307] RPC server started. Bound to: 127.30.68.2:43795
21:45:22.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.140067 31578 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.68.2:43795 every 8 connection(s)
21:45:22.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.142613 31443 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/data/info.pb
21:45:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.151330 30992 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31443
21:45:22.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.152189 30992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-1/wal/instance
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.171716 30992 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/wal
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/logs
21:45:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data/info.pb
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.3:0
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.68.3
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.3
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:45:22.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.211222 31580 heartbeater.cc:347] Connected to a master server at 127.30.68.61:37547
21:45:22.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.211285 31581 heartbeater.cc:347] Connected to a master server at 127.30.68.62:40071
21:45:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.211498 31579 heartbeater.cc:347] Connected to a master server at 127.30.68.60:46113
21:45:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.211870 31579 heartbeater.cc:464] Registering TS with master...
21:45:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.211877 31580 heartbeater.cc:464] Registering TS with master...
21:45:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.212026 31581 heartbeater.cc:464] Registering TS with master...
21:45:22.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.213532 31581 heartbeater.cc:511] Master 127.30.68.62:40071 requested a full tablet report, sending...
21:45:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.213603 31579 heartbeater.cc:511] Master 127.30.68.60:46113 requested a full tablet report, sending...
21:45:22.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.213564 31580 heartbeater.cc:511] Master 127.30.68.61:37547 requested a full tablet report, sending...
21:45:22.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.216763 31196 ts_manager.cc:194] Registered new tserver with Master: a6306970dc9a4c398c6958d62eb73b2d (127.30.68.2:43795)
21:45:22.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.217361 31114 ts_manager.cc:194] Registered new tserver with Master: a6306970dc9a4c398c6958d62eb73b2d (127.30.68.2:43795)
21:45:22.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.218550 31196 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.68.2:38127
21:45:22.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.221244 31037 ts_manager.cc:194] Registered new tserver with Master: a6306970dc9a4c398c6958d62eb73b2d (127.30.68.2:43795)
21:45:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.519057 31591 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.519598 31591 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:22.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.520062 31591 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:22.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.553218 31591 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:22.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.554069 31591 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.68.3
21:45:22.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.589416 31591 tablet_server_runner.cc:78] Tablet server non-default flags:
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.68.20:39247
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/wal
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.68.3:0
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data/info.pb
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.68.3
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.68.62:40071,127.30.68.61:37547,127.30.68.60:46113
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.68.3
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/logs
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:22.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:45:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:22.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.591002 31591 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:22.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.593195 31591 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:22.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.611316 31600 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.221802 31579 heartbeater.cc:503] Master 127.30.68.60:46113 was elected leader, sending a full tablet report...
21:45:23.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.611658 31598 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:22.611651 31597 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:22.612639 31591 server_base.cc:1034] running on GCE node
21:45:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.740703 31591 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.745226 31591 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:23.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.747005 31591 hybrid_clock.cc:648] HybridClock initialized: now 1714686323746965 us; error 39 us; skew 500 ppm
21:45:23.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.748407 31591 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.756609 31591 webserver.cc:460] Webserver started at http://127.30.68.3:36601/ using document root <none> and password file <none>
21:45:23.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.757896 31591 fs_manager.cc:359] Metadata directory not provided
21:45:23.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.758184 31591 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:23.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.758739 31591 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:23.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.766284 31591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data/instance:
21:45:23.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "51e5c7a121164f4680399d6c71087960"
21:45:23.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:23 on dist-test-slave-zqx8"
21:45:23.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.767882 31591 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/wal/instance:
21:45:23.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "51e5c7a121164f4680399d6c71087960"
21:45:23.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:23 on dist-test-slave-zqx8"
21:45:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.776969 31591 fs_manager.cc:691] Time spent creating directory manager: real 0.008s	user 0.007s	sys 0.001s
21:45:23.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.783187 31608 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:23.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.784482 31591 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.000s
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.784830 31591 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/wal
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "51e5c7a121164f4680399d6c71087960"
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:23 on dist-test-slave-zqx8"
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.785185 31591 fs_report.cc:383] FS layout report
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/wal
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/wal
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data/data
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:23.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.845526 31591 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.847169 31591 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:23.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.847623 31591 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:23.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.850448 31591 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
21:45:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.854997 31591 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:45:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.855226 31591 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
21:45:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.855432 31591 ts_tablet_manager.cc:590] Registered 0 tablets
21:45:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:23.855568 31591 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
21:45:24.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.008720 31591 rpc_server.cc:307] RPC server started. Bound to: 127.30.68.3:36431
21:45:24.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.008847 31725 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.68.3:36431 every 8 connection(s)
21:45:24.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.011432 31591 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/data/info.pb
21:45:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.018085 30992 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31591
21:45:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.018736 30992 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster6737921133403064978/ts-2/wal/instance
21:45:24.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.065538 31726 heartbeater.cc:347] Connected to a master server at 127.30.68.60:46113
21:45:24.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.066082 31726 heartbeater.cc:464] Registering TS with master...
21:45:24.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.067471 31728 heartbeater.cc:347] Connected to a master server at 127.30.68.62:40071
21:45:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.067849 31728 heartbeater.cc:464] Registering TS with master...
21:45:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.067507 31726 heartbeater.cc:511] Master 127.30.68.60:46113 requested a full tablet report, sending...
21:45:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.069099 31728 heartbeater.cc:511] Master 127.30.68.62:40071 requested a full tablet report, sending...
21:45:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.069494 31727 heartbeater.cc:347] Connected to a master server at 127.30.68.61:37547
21:45:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.069857 31727 heartbeater.cc:464] Registering TS with master...
21:45:24.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.070799 31727 heartbeater.cc:511] Master 127.30.68.61:37547 requested a full tablet report, sending...
21:45:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.071386 31196 ts_manager.cc:194] Registered new tserver with Master: 51e5c7a121164f4680399d6c71087960 (127.30.68.3:36431)
21:45:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.072362 31037 ts_manager.cc:194] Registered new tserver with Master: 51e5c7a121164f4680399d6c71087960 (127.30.68.3:36431)
21:45:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.073204 31196 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.30.68.3:42021
21:45:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.073901 31114 ts_manager.cc:194] Registered new tserver with Master: 51e5c7a121164f4680399d6c71087960 (127.30.68.3:36431)
21:45:24.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.094833 30992 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
21:45:24.106 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
21:45:24.107 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

21:45:24.118 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.30.68.62"
      port: 40071
    }
    bound_http_address {
      host: "127.30.68.62"
      port: 44547
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.30.68.61"
      port: 37547
    }
    bound_http_address {
      host: "127.30.68.61"
      port: 46675
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.30.68.60"
      port: 46113
    }
    bound_http_address {
      host: "127.30.68.60"
      port: 45005
    }
  }
}

21:45:24.125 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:45:24.128 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.30.68.1"
      port: 41599
    }
    bound_http_address {
      host: "127.30.68.1"
      port: 43923
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.30.68.2"
      port: 43795
    }
    bound_http_address {
      host: "127.30.68.2"
      port: 39839
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.30.68.3"
      port: 36431
    }
    bound_http_address {
      host: "127.30.68.3"
      port: 36601
    }
  }
}

21:45:24.129 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:45:24.346 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
21:45:24.482 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskAO3oum/test-tmp
21:45:24.483 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:45:24.483 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskAO3oum/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:45:24.498 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 0
  enable_kerberos: true
  cluster_root: "/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031"
  extra_master_flags: "--authn_token_validity_seconds=10"
  hms_mode: NONE
  mini_kdc_options {
    ticket_lifetime: "10s"
    renew_lifetime: "20s"
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

21:45:24.842 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:24.843 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}

21:45:24.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.846403 31737 mini_oidc.cc:152] Starting JWKS server
21:45:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.850502 31737 webserver.cc:460] Webserver started at http://127.0.0.1:36491/ using document root <none> and password file <none>
21:45:24.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.851145 31737 mini_oidc.cc:189] Starting OIDC Discovery server
21:45:24.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:24.853389 31737 webserver.cc:460] Webserver started at http://127.0.0.1:33589/ using document root <none> and password file <none>
21:45:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:45:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/principal' for realm 'KRBTEST.COM',
21:45:24.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:45:24.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:24 dist-test-slave-zqx8 krb5kdc[31749](info): setting up network...
21:45:24.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:45:24.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:24 dist-test-slave-zqx8 krb5kdc[31749](info): set up 2 sockets
21:45:24.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:24 dist-test-slave-zqx8 krb5kdc[31749](info): commencing operation
21:45:24.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:45:25.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:25.076961 31726 heartbeater.cc:503] Master 127.30.68.60:46113 was elected leader, sending a full tablet report...
21:45:25.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:25.190371 31070 debug-util.cc:398] Leaking SignalData structure 0x7b08000a0860 after lost signal to thread 31005
21:45:25.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:25.191835 31070 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4520 after lost signal to thread 31073
21:45:26.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:26.696305 31737 mini_kdc.cc:121] Time spent starting KDC: real 1.801s	user 0.009s	sys 0.013s
21:45:26.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:45:26.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:45:26.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:45:26.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:45:26.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:45:26.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:45:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:45:26.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:45:26.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:45:26.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:45:26.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/test-user.keytab.
21:45:26.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/test-user.keytab.
21:45:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:26 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686326, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:26.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
21:45:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:45:26Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
21:45:26.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:45:26Z Disabled control of system clock
21:45:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.254.126@KRBTEST.COM; defaulting to no policy
21:45:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:45:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.254.126@KRBTEST.COM" created.
21:45:26.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:45:26.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.254.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.126.keytab.
21:45:26.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.254.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.126.keytab.
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:26.919996 31737 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:26.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/logs
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/info.pb
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.126:36091
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.126
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.126.keytab
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.126
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:26.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:45:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.275646 31766 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:27.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.276228 31766 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:27.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.276911 31766 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.311435 31766 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:45:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.311691 31766 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.311961 31766 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.312151 31766 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.312336 31766 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.312587 31766 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:27.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.312745 31766 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36491/jwks.json
21:45:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:27.361409 31766 master_runner.cc:386] Master server non-default flags:
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.126.keytab
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.126
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.126:36091
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/info.pb
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.126
21:45:27.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/logs
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:27.362604 31766 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:27.363648 31766 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:27.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.382726 31775 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.384506 31772 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:28.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:27.383164 31773 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:28.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:27.386315 31766 server_base.cc:1034] running on GCE node
21:45:28.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.560195 31766 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:28.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.563442 31766 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.564893 31766 hybrid_clock.cc:648] HybridClock initialized: now 1714686328564860 us; error 47 us; skew 500 ppm
21:45:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.565692 31766 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:28 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686328, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.577486 31766 init.cc:377] Logged in from keytab as kudu/127.30.254.126@KRBTEST.COM (short username kudu)
21:45:28.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.583084 31766 webserver.cc:460] Webserver started at http://127.30.254.126:33069/ using document root <none> and password file <none>
21:45:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.584084 31766 fs_manager.cc:359] Metadata directory not provided
21:45:28.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.584313 31766 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:28.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.584844 31766 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.589748 31766 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/instance:
21:45:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9cef9425d094f7f87122a209141d693"
21:45:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:28 on dist-test-slave-zqx8"
21:45:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.590914 31766 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal/instance:
21:45:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9cef9425d094f7f87122a209141d693"
21:45:28.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:28 on dist-test-slave-zqx8"
21:45:28.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.598506 31766 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.001s	sys 0.008s
21:45:28.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.604041 31785 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:28.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.605304 31766 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.605727 31766 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9cef9425d094f7f87122a209141d693"
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:28 on dist-test-slave-zqx8"
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.606177 31766 fs_report.cc:383] FS layout report
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/data
21:45:28.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:28.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:28.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.658957 31766 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:28.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.664489 31766 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:28.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.665047 31766 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.740808 31766 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.126:36091
21:45:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.740999 31838 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.254.126:36091 every 8 connection(s)
21:45:28.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.744766 31766 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/info.pb
21:45:28.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.750918 31737 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31766
21:45:28.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.751479 31737 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal/instance
21:45:28.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.751667 31839 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:45:28.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.763921 31839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } has no permanent_uuid. Determining permanent_uuid...
21:45:28.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.254.125@KRBTEST.COM; defaulting to no policy
21:45:28.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:45:28.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.254.125@KRBTEST.COM" created.
21:45:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:28 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686328, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.126@KRBTEST.COM for kudu/127.30.254.126@KRBTEST.COM
21:45:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:45:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.254.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.125.keytab.
21:45:28.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.254.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.125.keytab.
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.806074 31737 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/logs
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/info.pb
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.125:32843
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.125
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.125.keytab
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.125
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:45:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.806345 31839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } has no permanent_uuid. Determining permanent_uuid...
21:45:28.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:28.809655 31786 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:28.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:28.814242 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.125:32843: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:28.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.863878 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } attempt: 1
21:45:28.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:28.867996 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.125:32843: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:28.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:28.936486 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } attempt: 2
21:45:28.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:28.942723 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.125:32843: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:29.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:29.034193 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } attempt: 3
21:45:29.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.038319 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.125:32843: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:29.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.133608 31846 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:29.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.134184 31846 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:29.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.134955 31846 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:29.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.168291 31846 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:45:29.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.168599 31846 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:29.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.168887 31846 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:29.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.169096 31846 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:29.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.169291 31846 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:29.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.169487 31846 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:29.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.169631 31846 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36491/jwks.json
21:45:29.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:29.184018 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } attempt: 4
21:45:29.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.189217 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.125:32843: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:29.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:29.214198 31846 master_runner.cc:386] Master server non-default flags:
21:45:29.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:29.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:29.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:29.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:29.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data
21:45:29.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.125.keytab
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.125
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.125:32843
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/info.pb
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.125
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/logs
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:29.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:29.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:29.215771 31846 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:29.217028 31846 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:29.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.235050 31852 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:29.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:29.488659 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } attempt: 5
21:45:29.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.495081 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.125:32843: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:30.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.042742 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } attempt: 6
21:45:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.050866 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.125:32843: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:30.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.235543 31853 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:30.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:29.236469 31846 server_base.cc:1034] running on GCE node
21:45:30.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:29.236869 31855 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:30.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.392051 31846 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:30.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.394856 31846 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:30.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.396214 31846 hybrid_clock.cc:648] HybridClock initialized: now 1714686330396162 us; error 72 us; skew 500 ppm
21:45:30.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.397125 31846 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:30 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686330, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.403641 31846 init.cc:377] Logged in from keytab as kudu/127.30.254.125@KRBTEST.COM (short username kudu)
21:45:30.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.409065 31846 webserver.cc:460] Webserver started at http://127.30.254.125:37477/ using document root <none> and password file <none>
21:45:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.409957 31846 fs_manager.cc:359] Metadata directory not provided
21:45:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.410145 31846 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:30.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.410621 31846 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:30.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.415732 31846 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/instance:
21:45:30.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e47f07cf757640d2876651b07a148766"
21:45:30.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:30 on dist-test-slave-zqx8"
21:45:30.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.416985 31846 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal/instance:
21:45:30.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e47f07cf757640d2876651b07a148766"
21:45:30.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:30 on dist-test-slave-zqx8"
21:45:30.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.425144 31846 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
21:45:30.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.430953 31865 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:30.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.432209 31846 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
21:45:30.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.432580 31846 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e47f07cf757640d2876651b07a148766"
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:30 on dist-test-slave-zqx8"
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.432945 31846 fs_report.cc:383] FS layout report
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/data
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:30.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.498615 31846 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.504768 31846 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.505300 31846 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:30.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.581415 31846 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.125:32843
21:45:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.581444 31918 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.254.125:32843 every 8 connection(s)
21:45:30.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.585820 31846 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/info.pb
21:45:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.589429 31737 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31846
21:45:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.589943 31737 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal/instance
21:45:30.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.592257 31919 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:45:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.254.124@KRBTEST.COM; defaulting to no policy
21:45:30.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.604233 31919 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } has no permanent_uuid. Determining permanent_uuid...
21:45:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:45:30.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.254.124@KRBTEST.COM" created.
21:45:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:30 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686330, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.125@KRBTEST.COM for kudu/127.30.254.126@KRBTEST.COM
21:45:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:45:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.254.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.124.keytab.
21:45:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.254.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.124.keytab.
21:45:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.635334 31737 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/logs
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/info.pb
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.124:33911
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.124
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.124.keytab
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.124
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:45:30.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.641305 31919 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } has no permanent_uuid. Determining permanent_uuid...
21:45:30.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:30 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686330, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.125@KRBTEST.COM for kudu/127.30.254.125@KRBTEST.COM
21:45:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.666666 31919 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } has no permanent_uuid. Determining permanent_uuid...
21:45:30.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.669035 31868 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:30.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.673208 31919 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:30.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.722743 31919 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 1
21:45:30.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.727607 31919 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:30.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.796066 31919 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 2
21:45:30.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.800153 31919 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:30.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:30.891539 31919 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 3
21:45:30.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.895593 31919 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:30.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.970606 31925 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:30.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.971194 31925 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:30.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:30.971781 31925 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.004436 31925 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:45:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.004736 31925 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.005100 31925 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.005331 31925 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.005530 31925 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.005726 31925 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.005872 31925 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36491/jwks.json
21:45:31.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.039088 31919 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 4
21:45:31.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.043037 31919 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.047228 31925 master_runner.cc:386] Master server non-default flags:
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.124.keytab
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.124
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.124:33911
21:45:31.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/info.pb
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.124
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/logs
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:31.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.048444 31925 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.049578 31925 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:31.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.067276 31936 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.111568 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } attempt: 7
21:45:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:31 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686328, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.126@KRBTEST.COM for kudu/127.30.254.125@KRBTEST.COM
21:45:31.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.170886 31839 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } has no permanent_uuid. Determining permanent_uuid...
21:45:31.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.182440 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.241115 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 1
21:45:31.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.245621 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.327335 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 2
21:45:31.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.333637 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.342432 31919 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 5
21:45:31.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.348153 31919 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.419063 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 3
21:45:31.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.425172 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.565611 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 4
21:45:31.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.571377 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.855032 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 5
21:45:31.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.860270 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.895602 31919 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 6
21:45:31.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.902973 31919 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.067618 31934 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:31.068132 31933 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:31.071883 31925 server_base.cc:1034] running on GCE node
21:45:32.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.259112 31925 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:32.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.262516 31925 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.263994 31925 hybrid_clock.cc:648] HybridClock initialized: now 1714686332263930 us; error 80 us; skew 500 ppm
21:45:32.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.264959 31925 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686332, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:32.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.271024 31925 init.cc:377] Logged in from keytab as kudu/127.30.254.124@KRBTEST.COM (short username kudu)
21:45:32.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.276356 31925 webserver.cc:460] Webserver started at http://127.30.254.124:37913/ using document root <none> and password file <none>
21:45:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.277344 31925 fs_manager.cc:359] Metadata directory not provided
21:45:32.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.277542 31925 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:32.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.278008 31925 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
21:45:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.285712 31925 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/instance:
21:45:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98965b4099cc40c680fe08a5c6e703ff"
21:45:32.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:32 on dist-test-slave-zqx8"
21:45:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.290454 31925 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal/instance:
21:45:32.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98965b4099cc40c680fe08a5c6e703ff"
21:45:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:32 on dist-test-slave-zqx8"
21:45:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.299055 31925 fs_manager.cc:691] Time spent creating directory manager: real 0.008s	user 0.004s	sys 0.004s
21:45:32.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.305930 31953 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.307193 31925 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.003s
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.307592 31925 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98965b4099cc40c680fe08a5c6e703ff"
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:32 on dist-test-slave-zqx8"
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.308013 31925 fs_report.cc:383] FS layout report
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/data
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:32.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:32.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.374341 31925 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:32.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.382061 31925 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:32.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.382704 31925 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:32.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.412976 31839 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } attempt: 6
21:45:32.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:32.421336 31839 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.30.254.124:33911: Network error: Client connection negotiation failed: client connection to 127.30.254.124:33911: connect: Connection refused (error 111)
21:45:32.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.458681 31925 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.124:33911
21:45:32.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.458768 32008 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.254.124:33911 every 8 connection(s)
21:45:32.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.461311 31925 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/info.pb
21:45:32.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.467082 32009 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:45:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.471067 31737 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 31925
21:45:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.471505 31737 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal/instance
21:45:32.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.481369 32009 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } has no permanent_uuid. Determining permanent_uuid...
21:45:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686332, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for kudu/127.30.254.126@KRBTEST.COM
21:45:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686326, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.254.124@KRBTEST.COM
21:45:32.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.534088 32009 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } has no permanent_uuid. Determining permanent_uuid...
21:45:32.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686332, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for kudu/127.30.254.125@KRBTEST.COM
21:45:32.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686326, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.254.125@KRBTEST.COM
21:45:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.558965 32009 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } has no permanent_uuid. Determining permanent_uuid...
21:45:32.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686332, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for kudu/127.30.254.124@KRBTEST.COM
21:45:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686326, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.254.126@KRBTEST.COM
21:45:32.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.589797 32009 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Bootstrap starting.
21:45:32.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.591856 31737 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
21:45:32.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.597990 32009 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Neither blocks nor log segments found. Creating new log.
21:45:32.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:32.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.600788 32009 log.cc:824] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Log is configured to *not* fsync() on all Append() calls
21:45:32.601 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}

21:45:32.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.606899 32009 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: No bootstrap required, opened a new log
21:45:32.619 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
  env_vars {
    key: "KRB5CCNAME"
    value: "/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc"
  }
  env_vars {
    key: "KRB5_CONFIG"
    value: "/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5.conf"
  }
  env_vars {
    key: "KRB5_KDC_PROFILE"
    value: "/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kdc.conf"
  }
  env_vars {
    key: "KUDU_ENABLE_KRB5_REALM_FIX"
    value: "yes"
  }
}

21:45:32.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}

21:45:32.623 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.30.254.126"
      port: 36091
    }
    bound_http_address {
      host: "127.30.254.126"
      port: 33069
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.30.254.125"
      port: 32843
    }
    bound_http_address {
      host: "127.30.254.125"
      port: 37477
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.30.254.124"
      port: 33911
    }
    bound_http_address {
      host: "127.30.254.124"
      port: 37913
    }
  }
}

21:45:32.623 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}

21:45:32.624 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:32.624 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:45:32.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

21:45:32.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.638535 32009 raft_consensus.cc:348] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:32.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.639492 32009 raft_consensus.cc:374] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:45:32.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.639782 32009 raft_consensus.cc:729] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 98965b4099cc40c680fe08a5c6e703ff, State: Initialized, Role: FOLLOWER
21:45:32.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.641012 32009 consensus_queue.cc:260] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.647603 32021 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } } }
21:45:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.648684 32021 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [sys.catalog]: This master's current role is: FOLLOWER
21:45:32.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.649832 32009 sys_catalog.cc:564] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [sys.catalog]: configured and running, proceeding with master startup.
21:45:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686332, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:32.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
21:45:32.668 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:32.670 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc
21:45:32.680 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:45:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:32.680891 32033 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:32.681 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc
21:45:32.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:32.681524 32033 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:32.682 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:45:32.687 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:45:32.689 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:45:32.690 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

21:45:32.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.675356 31925 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: OK, instance_id { permanent_uuid: "a9cef9425d094f7f87122a209141d693" instance_seqno: 1714686328718613 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:45:32.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.701874 31925 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:32.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:32 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686332, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:32.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
21:45:32.736 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:32.736 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc
21:45:32.738 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:45:32.739 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.254.124:33911
21:45:32.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

21:45:32.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.742233 31737 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31925
21:45:32.767 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:32.768 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.254.126:36091
21:45:32.768 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

21:45:32.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.768807 31737 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31766
21:45:32.788 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:32.788 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.254.125:32843
21:45:32.789 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

21:45:32.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.789434 31737 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31846
21:45:32.813 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:32.813 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.254.124:33911
21:45:32.815 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 2
  }
}

21:45:32.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:32.816640 31737 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:32.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:32.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:32.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data
21:45:32.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/logs
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/info.pb
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.124:33911
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.124
21:45:32.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.124.keytab
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.124
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:32.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:45:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.147686 32037 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.148391 32037 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.149291 32037 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:33.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.185691 32037 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:45:33.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.185988 32037 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:33.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.186401 32037 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:33.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.186724 32037 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.187023 32037 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.187325 32037 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.187549 32037 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36491/jwks.json
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:33.227540 32037 master_runner.cc:386] Master server non-default flags:
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.124.keytab
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.124
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:33.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.124:33911
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/info.pb
21:45:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.124
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37913
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/logs
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:33.229202 32037 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:33.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:33.230751 32037 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:33.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.247035 32043 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:34.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.247352 32044 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:34.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:33.249085 32037 server_base.cc:1034] running on GCE node
21:45:34.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:33.248004 32046 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.404340 32037 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.407312 32037 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.408728 32037 hybrid_clock.cc:648] HybridClock initialized: now 1714686334408674 us; error 59 us; skew 500 ppm
21:45:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.409663 32037 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:34.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:34 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686334, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.421720 32037 init.cc:377] Logged in from keytab as kudu/127.30.254.124@KRBTEST.COM (short username kudu)
21:45:34.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.427367 32037 webserver.cc:460] Webserver started at http://127.30.254.124:37913/ using document root <none> and password file <none>
21:45:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.428400 32037 fs_manager.cc:359] Metadata directory not provided
21:45:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.428714 32037 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:34.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.437407 32037 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.007s	sys 0.001s
21:45:34.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.442041 32056 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.443202 32037 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.004s
21:45:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.443574 32037 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "98965b4099cc40c680fe08a5c6e703ff"
21:45:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:32 on dist-test-slave-zqx8"
21:45:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.446158 32037 fs_report.cc:383] FS layout report
21:45:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/wal
21:45:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/data
21:45:34.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.510636 32037 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:34.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.517706 32037 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:34.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.518221 32037 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:34.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.591775 32037 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.124:33911
21:45:34.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.591926 32109 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.254.124:33911 every 8 connection(s)
21:45:34.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.594738 32037 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-2/data/info.pb
21:45:34.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.596139 31737 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 32037
21:45:34.596 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:34.597 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.254.126:36091
21:45:34.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 0
  }
}

21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.598981 31737 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:34.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/logs
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/info.pb
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.126:36091
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.126
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33069
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.126.keytab
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.126
21:45:34.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:45:34.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.607697 32110 sys_catalog.cc:263] Verifying existing consensus state
21:45:34.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.613348 32110 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Bootstrap starting.
21:45:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.648885 32110 log.cc:824] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Log is configured to *not* fsync() on all Append() calls
21:45:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.653574 32110 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:45:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.654255 32110 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Bootstrap complete.
21:45:34.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.685176 32110 raft_consensus.cc:348] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:34.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.689633 32110 raft_consensus.cc:374] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:45:34.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.690045 32110 raft_consensus.cc:729] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 98965b4099cc40c680fe08a5c6e703ff, State: Initialized, Role: FOLLOWER
21:45:34.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.691409 32110 consensus_queue.cc:260] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:34.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.694676 32114 sys_catalog.cc:455] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } } }
21:45:34.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.695613 32114 sys_catalog.cc:458] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [sys.catalog]: This master's current role is: FOLLOWER
21:45:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.696482 32110 sys_catalog.cc:564] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [sys.catalog]: configured and running, proceeding with master startup.
21:45:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:34.726660 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:34.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:34.727043 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:34.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.732530 32037 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111), 
21:45:34.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:34.771064 32037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:34.773697 32063 tablet.cc:2369] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:45:34.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:34.975986 32112 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:34.976713 32112 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:34.980621 32112 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.022368 32112 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:45:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.022652 32112 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.023087 32112 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.023407 32112 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.023705 32112 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.024009 32112 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.024230 32112 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36491/jwks.json
21:45:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.065268 32112 master_runner.cc:386] Master server non-default flags:
21:45:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:35.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.126.keytab
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.126
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:35.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.126:36091
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/info.pb
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.126
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33069
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/logs
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:35.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:35.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:35.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.066851 32112 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:35.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.068409 32112 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:35.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.092576 32137 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.126137 32114 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:45:35.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.126777 32114 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:35.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.133134 32114 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:45:35.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.134323 32060 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:45:35.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.169891 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:45:35.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.170773 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:35.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.171319 32060 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:45:35.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:35.185081 32114 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:45:35.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.733887 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:35.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.734457 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:36.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:35.092592 32136 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:36.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:36.619347 32139 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:36.619788 32138 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1525 milliseconds
21:45:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.619812 32112 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:45:36.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.626209 32112 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:36.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.629537 32112 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:36.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.631034 32112 hybrid_clock.cc:648] HybridClock initialized: now 1714686336630837 us; error 202 us; skew 500 ppm
21:45:36.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.631961 32112 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:36.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:36 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686336, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:36.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.638638 32112 init.cc:377] Logged in from keytab as kudu/127.30.254.126@KRBTEST.COM (short username kudu)
21:45:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.644754 32112 webserver.cc:460] Webserver started at http://127.30.254.126:33069/ using document root <none> and password file <none>
21:45:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.645844 32112 fs_manager.cc:359] Metadata directory not provided
21:45:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.646214 32112 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:36.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.655370 32112 fs_manager.cc:709] Time spent opening directory manager: real 0.006s	user 0.001s	sys 0.004s
21:45:36.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.660679 32149 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.662081 32112 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.000s
21:45:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.662539 32112 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9cef9425d094f7f87122a209141d693"
21:45:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:28 on dist-test-slave-zqx8"
21:45:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.666203 32112 fs_report.cc:383] FS layout report
21:45:36.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/data
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:36.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:36.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.737828 32112 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:36.738763 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:36.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:36.739109 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.746935 32112 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.747558 32112 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:36.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.840899 32112 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.126:36091
21:45:36.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.840930 32200 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.254.126:36091 every 8 connection(s)
21:45:36.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.845326 32112 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/data/info.pb
21:45:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.850135 31737 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 32112
21:45:36.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:36.852 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.254.125:32843
21:45:36.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 1
  }
}

21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.855194 31737 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu
21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data
21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:45:36.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/logs
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/info.pb
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.125:32843
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.125
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37477
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.125.keytab
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.125
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:36.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:36.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:36.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:45:36.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.861231 32201 sys_catalog.cc:263] Verifying existing consensus state
21:45:36.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 21:45:36.862401 32201 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:45:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.863435 32112 master.cc:537] Master@127.30.254.126:36091 shutting down...
21:45:36.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:36.947188 32112 master.cc:552] Master@127.30.254.126:36091 shutdown complete.
21:45:36.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:45:37.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.234050 32203 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:45:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.234855 32203 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:45:37.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.235507 32203 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:45:37.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.269443 32203 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:45:37.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.269698 32203 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:45:37.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.270010 32203 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:45:37.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.270270 32203 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:45:37.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.270570 32203 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:45:37.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.270891 32203 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:45:37.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.271143 32203 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36491/jwks.json
21:45:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.311350 32203 master_runner.cc:386] Master server non-default flags:
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.254.84:40125
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.254.126:36091,127.30.254.125:32843,127.30.254.124:33911
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/krb5kdc/kudu_127.30.254.125.keytab
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.254.125
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.254.125:32843
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36491/jwks.json
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/info.pb
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.254.125
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37477
21:45:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/logs
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision fdb42489d76098eeca371ac57ae02ea176438c64
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 May 2024 21:09:19 UTC on efd1337a2aa3
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 96
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.312618 32203 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:45:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.313632 32203 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:45:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.336690 32211 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:37.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.427381 32216 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:45:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.427815 32216 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:37.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.429862 32216 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:45:37.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.438750 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.439314 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:45:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.439718 32060 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:45:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.441227 32216 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:45:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.743206 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.743575 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.336855 32210 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:37.338034 32203 server_base.cc:1034] running on GCE node
21:45:38.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:37.338218 32213 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:45:38.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.541417 32203 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:45:38.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.544245 32203 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:45:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.545689 32203 hybrid_clock.cc:648] HybridClock initialized: now 1714686338545639 us; error 61 us; skew 500 ppm
21:45:38.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.546686 32203 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:45:38.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:38 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686338, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.553006 32203 init.cc:377] Logged in from keytab as kudu/127.30.254.125@KRBTEST.COM (short username kudu)
21:45:38.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.558288 32203 webserver.cc:460] Webserver started at http://127.30.254.125:37477/ using document root <none> and password file <none>
21:45:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.559207 32203 fs_manager.cc:359] Metadata directory not provided
21:45:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.559417 32203 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:45:38.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.567523 32203 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
21:45:38.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.572013 32227 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
21:45:38.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.573253 32203 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.573578 32203 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data,/tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e47f07cf757640d2876651b07a148766"
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-02 21:45:30 on dist-test-slave-zqx8"
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.575958 32203 fs_report.cc:383] FS layout report
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/data
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:45:38.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:45:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:45:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:45:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:45:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:45:38.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.639103 32203 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
21:45:38.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.644734 32203 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:45:38.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.645206 32203 kserver.cc:163] Server-wide thread pool size limit: 3276
21:45:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.720293 32203 rpc_server.cc:307] RPC server started. Bound to: 127.30.254.125:32843
21:45:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.720384 32280 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.30.254.125:32843 every 8 connection(s)
21:45:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.723575 32203 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/data/info.pb
21:45:38.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
21:45:38.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.725998 31737 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu as pid 32203
21:45:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.737836 32281 sys_catalog.cc:263] Verifying existing consensus state
21:45:38.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240502 21:45:38.738878 32281 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:45:38.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.739424 32203 master.cc:537] Master@127.30.254.125:32843 shutting down...
21:45:38.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:38.748237 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:38.749755 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:38.791 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.254.124:33911(127.30.254.124:33911)] connecting to peer
21:45:38.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:38.843489 32203 master.cc:552] Master@127.30.254.125:32843 shutdown complete.
21:45:38.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskAO3oum/test-tmp/mini-kudu-cluster8998118464134646031/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
21:45:38.860 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:38.866 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:38.915 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.254.124:33911(127.30.254.124:33911)] Successfully connected to peer
21:45:38.917 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:38.917 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:38.919 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:38.977 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:38.978 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:38.979 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:38.996 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:45:39.013 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.254.124
21:45:39.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:39 dist-test-slave-zqx8 krb5kdc[31749](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686332, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.254.124@KRBTEST.COM
21:45:39.231 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x34cf7eef, L:/127.0.0.1:57882 - R:/127.30.254.124:33911] using SASL/GSSAPI
21:45:39.247 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.259 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.260 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.260 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.261 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.263 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.263 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.264 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.264 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.274 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.275 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.276 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.276 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.282 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.284 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.285 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.285 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.285 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.294 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.296 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.299 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.300 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.300 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.301 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.334 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.337 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.337 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.337 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.338 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.340 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.353 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.355 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.356 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.357 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.357 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.360 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.360 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.360 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.361 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.376 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.377 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.377 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.378 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.379 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.379 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.380 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.515 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.516 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.520 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.520 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.713 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:39.715 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:39.715 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.716 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.717 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:39.720 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.720 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.720 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:39.721 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:39.754173 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:39.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:39.754534 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:39.774691 32037 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111), 
21:45:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:39.775074 32037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:40.133 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:40.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:40.136 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:40.136 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:40.137 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:40.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:40.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:40.139 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:40.140 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:40.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:40.231297 32289 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:45:40.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:40.232139 32289 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:40.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:40.235023 32289 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:45:40.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:40.236474 32060 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:45:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:40.240515 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:45:40.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:40.243667 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:40.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:40.244011 32058 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:45:40.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:40.244738 32289 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:45:40.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:40.757419 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:40.757742 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:40.874 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:40.876 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:40.877 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:40.877 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:40.877 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:40.879 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:40.879 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:40.879 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:40.879 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:41.222631 31070 debug-util.cc:398] Leaking SignalData structure 0x7b080009e240 after lost signal to thread 31005
21:45:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:41.223336 31070 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6880 after lost signal to thread 31073
21:45:41.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:41.760782 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:41.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:41.761091 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:42.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:42.580940 32298 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:45:42.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:42.581398 32298 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:42.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:42.583122 32298 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:45:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:42.588130 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:45:42.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:42.588544 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:42.588891 32058 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:45:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:42.589418 32298 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:45:42.634 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:42.636 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:42.636 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:42.636 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:42.637 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:42.639 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:42.639 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:42.639 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:42.639 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:42.763865 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:42.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:42.764169 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:43.373 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:43.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:43.377 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:43.377 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:43.377 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:43.377 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:43.377 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:43.377 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:43.379 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:43.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:43.767494 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:43.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:43.767870 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:44.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:44 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686344, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:44.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:44.425877 32052 init.cc:295] Successfully reacquired a new kerberos TGT
21:45:44.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:44.770691 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:44.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:44.770977 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:44.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:44.778499 32037 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111), 
21:45:44.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:44.778966 32037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:45.354 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:45.356 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:45.356 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:45.356 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:45.356 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:45.358 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:45.358 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:45.358 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:45.358 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:45.773921 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:45.774247 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:46.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:46.777047 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:46.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:46.777354 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:47.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:47.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:47.116 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:47.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:47.117 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:47.120 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:47.120 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:47.120 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:47.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:47.247283 32308 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:45:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:47.247818 32308 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:47.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:47.249662 32308 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:45:47.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:47.252310 32060 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:45:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:47.255571 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:45:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:47.256189 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:47.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:47.256613 32058 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:45:47.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:47.257180 32308 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:45:47.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:47.780061 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:47.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:47.780341 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:48.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:48.783346 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:48.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:48.783653 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:49.781896 32037 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111), 
21:45:49.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:49.782260 32037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:49.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:49.786473 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:49.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:49.786782 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:50.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:50.715378 32315 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:45:50.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:50.716609 32315 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:45:50.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:50.718259 32315 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:45:50.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:50.723189 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:45:50.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:50.723589 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:45:50.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:50.723927 32060 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:45:50.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:50.724527 32315 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:45:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:50.789767 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:50.790036 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:50.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:50.895 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:50.897 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:50.897 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:50.897 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:50.898 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:50.898 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:50.899 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:50.899 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:51.793203 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:51.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:51.793527 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:52.754 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:52.756 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:52.756 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:52.756 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:52.756 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:52.758 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:52.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:52.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:52.759 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:52.796826 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:52.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:52.797292 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:53.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:53.801041 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:53.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:53.801378 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:54.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:45:54 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686354, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:45:54.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:54.428366 32052 init.cc:295] Successfully reacquired a new kerberos TGT
21:45:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:54.785882 32037 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111), 
21:45:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:54.786350 32037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:54.804230 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:54.804585 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:55.807605 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:55.807883 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:56.234 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:45:56.235 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:45:56.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:56.237 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:56.237 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:45:56.238 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:56.238 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:56.238 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:45:56.239 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:45:56.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:56.810672 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:56.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:56.810958 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:57.813980 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:57.814266 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:58.817312 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:58.817647 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:45:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:59.789187 32037 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111), 
21:45:59.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:45:59.789549 32037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:45:59.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:59.820398 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:45:59.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:45:59.820750 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:00.235 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:46:00.237 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:00.237 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:00.237 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:00.238 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:46:00.239 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:00.239 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:00.239 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:00.240 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:46:00.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:00.823576 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:00.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:00.823872 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:01.826728 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:01.827034 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:02.830034 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:02.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:02.830360 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:03.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:46:03.015 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:46:03.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:03.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:03.017 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:03.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:03.017 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:03.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:03.018 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:46:03.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:03.669010 32339 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:46:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:03.669621 32339 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:46:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:03.671528 32339 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:46:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:03.675298 32058 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111) [suppressed 3 similar messages]
21:46:03.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:03.678772 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:46:03.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:03.679337 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:46:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:03.679754 32060 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:46:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:03.680536 32339 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:46:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:03.833776 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:03.834079 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:04.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:46:04.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:46:04.095 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:04.095 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:04.095 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:04.096 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:04.096 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:04.096 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:04.098 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:46:04.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 02 21:46:04 dist-test-slave-zqx8 krb5kdc[31749](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1714686364, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.254.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:46:04.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:04.430857 32052 init.cc:295] Successfully reacquired a new kerberos TGT
21:46:04.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:04.793496 32037 master_runner.cc:186] Error getting master registration for 127.30.254.126:36091: Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111), 
21:46:04.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:04.793970 32037 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:46:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:04.836825 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:04.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:04.837095 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:05.214 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:46:05.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:46:05.215 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:05.215 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:05.216 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:05.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:05.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:05.216 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:05.218 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:46:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:05.576429 32346 raft_consensus.cc:482] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:46:05.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:05.576915 32346 raft_consensus.cc:504] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a9cef9425d094f7f87122a209141d693" member_type: VOTER last_known_addr { host: "127.30.254.126" port: 36091 } } peers { permanent_uuid: "e47f07cf757640d2876651b07a148766" member_type: VOTER last_known_addr { host: "127.30.254.125" port: 32843 } } peers { permanent_uuid: "98965b4099cc40c680fe08a5c6e703ff" member_type: VOTER last_known_addr { host: "127.30.254.124" port: 33911 } }
21:46:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:05.578506 32346 leader_election.cc:288] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091), e47f07cf757640d2876651b07a148766 (127.30.254.125:32843)
21:46:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:05.582960 32060 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a9cef9425d094f7f87122a209141d693 (127.30.254.126:36091): Network error: Client connection negotiation failed: client connection to 127.30.254.126:36091: connect: Connection refused (error 111)
21:46:05.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:05.583640 32058 leader_election.cc:334] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e47f07cf757640d2876651b07a148766 (127.30.254.125:32843): Network error: Client connection negotiation failed: client connection to 127.30.254.125:32843: connect: Connection refused (error 111)
21:46:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:05.583969 32058 leader_election.cc:302] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 98965b4099cc40c680fe08a5c6e703ff; no voters: a9cef9425d094f7f87122a209141d693, e47f07cf757640d2876651b07a148766
21:46:05.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:05.584625 32346 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
21:46:05.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:05.839516 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:05.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:05.839802 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:06.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:06.842293 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:06.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:06.842561 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:07.845072 32126 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 98965b4099cc40c680fe08a5c6e703ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:46:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240502 21:46:07.845389 32126 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:46:08.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.126:36091(127.30.254.126:36091)] connecting to peer
21:46:08.015 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.30.254.125:32843(127.30.254.125:32843)] connecting to peer
21:46:08.016 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:08.016 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:08.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.254.126:36091(127.30.254.126:36091)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:08.016 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.125:32843(127.30.254.125:32843)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:08.016 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.125:32843: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843
21:46:08.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.254.126:36091: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
21:46:08.018 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843; will retry
21:46:08.022 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29286), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [242ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [507ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [511ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [524ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [525ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [529ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [541ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [548ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [549ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [550ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [562ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [563ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [565ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [601ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [602ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [603ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [621ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [623ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [625ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [643ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [645ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [782ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [785ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [980ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [983ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [985ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1404ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2137ms] refreshing cache from master, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [2141ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [2142ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2143ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3897ms] refreshing cache from master, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [3903ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3903ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [4637ms] refreshing cache from master, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [4638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [4641ms] too many traces: truncated at 100 traces, deferred=Deferred@801273186(state=PENDING, result=null, callback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062), errback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062)))
21:46:08.031 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.254.126:36091(127.30.254.126:36091)] disconnecting while in state TERMINATED
21:46:08.033 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.254.125:32843(127.30.254.125:32843)] disconnecting while in state TERMINATED
21:46:08.033 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.254.124:33911(127.30.254.124:33911)] disconnecting while in state READY
21:46:08.039 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:46:08.041 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.254.124:33911(127.30.254.124:33911)] handling channelInactive
21:46:08.041 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.254.124:33911(127.30.254.124:33911)] cleaning up while in state READY due to: connection closed
21:46:08.044 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:46:08.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:08.051173 31737 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 32037
21:46:09.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:46:09Z chronyd exiting
21:46:10.194 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:46:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:10.196427 30992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31290
21:46:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:10.225601 30992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31443
21:46:10.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:10.255077 30992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31591
21:46:10.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:10.289399 30992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31004
21:46:10.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:10.324918 30992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31076
21:46:10.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240502 21:46:10.360733 30992 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskAO3oum/build/tsan/bin/kudu with pid 31152
21:46:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-02T21:46:11Z chronyd exiting
21:46:12.480 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29286), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [242ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [507ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [511ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [524ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [525ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [529ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [541ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [548ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [549ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [550ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [562ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [563ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [565ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [601ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [602ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [603ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [621ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [623ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [625ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [643ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [645ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [782ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [785ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [980ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [983ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [985ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1404ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2137ms] refreshing cache from master, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [2141ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [2142ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2143ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3897ms] refreshing cache from master, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [3903ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3903ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [4637ms] refreshing cache from master, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [4638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [4641ms] too many traces: truncated at 100 traces, deferred=Deferred@801273186(state=PENDING, result=null, callback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062), errback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.254.126:36091
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29286), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [242ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [507ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [511ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [524ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [525ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [529ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [541ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [548ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [549ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [550ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [562ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [563ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [565ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [601ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [602ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [603ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [621ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [623ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [625ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [643ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [645ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [782ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [785ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [980ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [983ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [985ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1404ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2137ms] refreshing cache from master, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [2141ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [2142ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2143ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3897ms] refreshing cache from master, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [3903ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3903ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [4637ms] refreshing cache from master, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [4638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [4641ms] too many traces: truncated at 100 traces, deferred=Deferred@801273186(state=PENDING, result=null, callback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062), errback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.254.126:36091
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

21:46:13.036 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
21:46:13.037 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29286), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [242ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [507ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [511ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [524ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [525ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [529ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [541ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [548ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [549ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [550ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [562ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [563ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [565ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [601ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [602ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [603ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [621ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [623ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [625ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [643ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [645ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [782ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [785ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [980ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [983ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [985ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1404ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2137ms] refreshing cache from master, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [2141ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [2142ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2143ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3897ms] refreshing cache from master, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [3903ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3903ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [4637ms] refreshing cache from master, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [4638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [4641ms] too many traces: truncated at 100 traces, deferred=Deferred@801273186(state=PENDING, result=null, callback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062), errback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.254.126:36091
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=29286), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [182ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [242ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [507ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [511ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [517ms] refreshing cache from master, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [523ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [524ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [525ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [527ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [529ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [537ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [541ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [548ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [549ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [550ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [557ms] refreshing cache from master, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [562ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [563ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [564ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [565ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [601ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [602ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [603ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [604ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [621ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [623ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [624ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [625ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [641ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [643ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [645ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [777ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [782ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [783ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [784ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [785ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [977ms] refreshing cache from master, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [980ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [983ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [984ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [985ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [1402ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [1404ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2137ms] refreshing cache from master, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [2138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [2139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [2141ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [2142ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [2143ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [2143ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3897ms] refreshing cache from master, [3897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [3898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [3900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.126:36091: Network error: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091, [3901ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.124:33911: OK, [3903ms] Sub RPC ConnectToMaster: received response from server master-127.30.254.125:32843: Network error: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [3903ms] delaying RPC due to: Service unavailable: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843, [4637ms] refreshing cache from master, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.124:33911, [4637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.126:36091, [4638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.254.125:32843, [4641ms] too many traces: truncated at 100 traces, deferred=Deferred@801273186(state=PENDING, result=null, callback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062), errback=(continuation of Deferred@138471124 after retry RPC after error@328549504) -> (continuation of Deferred@51803800 after retry RPC after error@1069775613) -> (continuation of Deferred@809131415 after retry RPC after error@154316372) -> (continuation of Deferred@958442332 after retry RPC after error@495987223) -> (continuation of Deferred@2141457111 after retry RPC after error@1677216633) -> (continuation of Deferred@1606317585 after retry RPC after error@1130508131) -> (continuation of Deferred@483354474 after retry RPC after error@1036862473) -> (continuation of Deferred@292528991 after retry RPC after error@1126469414) -> (continuation of Deferred@615990303 after retry RPC after error@1938661772) -> (continuation of Deferred@529830393 after retry RPC after error@2024693906) -> (continuation of Deferred@1722242084 after retry RPC after error@2025330877) -> (continuation of Deferred@1134052618 after retry RPC after error@1220085531) -> (continuation of Deferred@350452828 after retry RPC after error@749138829) -> (continuation of Deferred@756503302 after retry RPC after error@756418656) -> (continuation of Deferred@572030978 after retry RPC after error@960050519) -> (continuation of Deferred@773967381 after retry RPC after error@862557881) -> (continuation of Deferred@1427576902 after retry RPC after error@636750910) -> (continuation of Deferred@92618739 after retry RPC after error@2049359268) -> (continuation of Deferred@387862414 after retry RPC after error@302920879) -> (continuation of Deferred@1046014906 after retry RPC after error@1910291016) -> (continuation of Deferred@349956546 after retry RPC after error@1997737146) -> (continuation of Deferred@250559705 after retry RPC after error@443931062)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.30.254.124:33911,127.30.254.126:36091,127.30.254.125:32843) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.125:32843(127.30.254.125:32843): Connection refused: /127.30.254.125:32843,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.30.254.126:36091(127.30.254.126:36091): Connection refused: /127.30.254.126:36091
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.30.254.126:36091
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)