Unable to diagnose

08:17:01.376 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:17:01.393 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskN8qzGC/test-tmp
08:17:01.396 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:17:01.401 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskN8qzGC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:17:01.686 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

08:17:01.825 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:17:01.826 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

08:17:01.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:01.829841 32220 mini_oidc.cc:152] Starting JWKS server
08:17:01.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:01.833424 32220 webserver.cc:460] Webserver started at http://127.0.0.1:39843/ using document root <none> and password file <none>
08:17:01.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:01.833873 32220 mini_oidc.cc:189] Starting OIDC Discovery server
08:17:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:01.836138 32220 webserver.cc:460] Webserver started at http://127.0.0.1:38927/ using document root <none> and password file <none>
08:17:01.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-07T08:17:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:17:01.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-07T08:17:01Z Disabled control of system clock
08:17:01.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:01.901808 32220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:01.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:01.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/wal
08:17:01.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data
08:17:01.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/logs
08:17:01.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data/info.pb
08:17:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:01.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.62:45213
08:17:01.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.62
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:01.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:01.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39843/jwks.json with env {}
08:17:02.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.216401 32232 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:02.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.217010 32232 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:02.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.217643 32232 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.248930 32232 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.249235 32232 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:02.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.249428 32232 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.249682 32232 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.249986 32232 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.250218 32232 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39843/jwks.json
08:17:02.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:02.286474 32232 master_runner.cc:386] Master server non-default flags:
08:17:02.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:02.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:02.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:02.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:02.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data
08:17:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/wal
08:17:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:02.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.62:45213
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39843/jwks.json
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data/info.pb
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.62
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:02.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/logs
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:02.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:02.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:02.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:02.287896 32232 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:02.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:02.290287 32232 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:02.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.307181 32238 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:03.707562 32237 debug-util.cc:398] Leaking SignalData structure 0x7b0800014880 after lost signal to thread 32232
08:17:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:03.787214 32237 kernel_stack_watchdog.cc:198] Thread 32232 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 400ms:
08:17:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:17:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:17:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:17:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:17:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:03.787617 32232 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.483s	user 0.460s	sys 1.014s
08:17:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:03.788038 32232 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.483s	user 0.460s	sys 1.014s
08:17:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:02.307381 32239 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:03.788795 32240 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1481 milliseconds
08:17:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.790141 32232 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:17:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:03.790169 32241 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.794528 32232 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:03.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.797379 32232 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:03.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.798808 32232 hybrid_clock.cc:648] HybridClock initialized: now 1715069823798763 us; error 66 us; skew 500 ppm
08:17:03.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.799854 32232 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:03.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.806545 32232 webserver.cc:460] Webserver started at http://127.31.119.62:40975/ using document root <none> and password file <none>
08:17:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.807638 32232 fs_manager.cc:359] Metadata directory not provided
08:17:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.807858 32232 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.808287 32232 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:03.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.813452 32232 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data/instance:
08:17:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34153164590348b8bd9c60db1d31c090"
08:17:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:03 on dist-test-slave-gbgz"
08:17:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.814584 32232 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/wal/instance:
08:17:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34153164590348b8bd9c60db1d31c090"
08:17:03.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:03 on dist-test-slave-gbgz"
08:17:03.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.822275 32232 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.005s
08:17:03.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.828397 32248 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:03.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.829613 32232 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
08:17:03.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.830001 32232 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/wal
08:17:03.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "34153164590348b8bd9c60db1d31c090"
08:17:03.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:03 on dist-test-slave-gbgz"
08:17:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.830446 32232 fs_report.cc:383] FS layout report
08:17:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/wal
08:17:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/wal
08:17:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data/data
08:17:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:03.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:03.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.892663 32232 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:03.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.894315 32232 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:03.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.894865 32232 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.966548 32232 rpc_server.cc:307] RPC server started. Bound to: 127.31.119.62:45213
08:17:03.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.966686 32299 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.31.119.62:45213 every 8 connection(s)
08:17:03.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.969575 32232 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/data/info.pb
08:17:03.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.973274 32220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 32232
08:17:03.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.973852 32220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-0/wal/instance
08:17:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.976222 32300 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:17:03.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.979101 32220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:03.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:03.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/wal
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/logs
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data/info.pb
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:03.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:03.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:03.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:03.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.61:43563
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.61
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:03.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39843/jwks.json with env {}
08:17:03.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:03.991437 32300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
08:17:04.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.017586 32300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } has no permanent_uuid. Determining permanent_uuid...
08:17:04.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.020226 32251 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:04.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.024403 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.61:43563: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:04.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.073865 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } attempt: 1
08:17:04.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.077647 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.61:43563: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.146243 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } attempt: 2
08:17:04.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.150590 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.61:43563: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:04.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.242157 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } attempt: 3
08:17:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.246210 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.61:43563: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:04.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.296471 32302 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:04.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.297057 32302 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:04.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.297729 32302 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.329278 32302 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.329566 32302 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.329759 32302 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.329957 32302 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.330140 32302 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:04.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.330340 32302 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39843/jwks.json
08:17:04.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.365983 32302 master_runner.cc:386] Master server non-default flags:
08:17:04.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:04.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/wal
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.61:43563
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:04.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39843/jwks.json
08:17:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data/info.pb
08:17:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.61
08:17:04.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/logs
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:04.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:04.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:04.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.367473 32302 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:04.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.369829 32302 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:04.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.386799 32314 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:04.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.389680 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } attempt: 4
08:17:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.394351 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.61:43563: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:04.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.693796 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } attempt: 5
08:17:04.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.699651 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.61:43563: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:05.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.247092 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } attempt: 6
08:17:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:05.255662 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.61:43563: Network error: Client connection negotiation failed: client connection to 127.31.119.61:43563: connect: Connection refused (error 111)
08:17:05.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.387303 32312 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:05.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:04.389880 32311 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:05.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:04.389950 32302 server_base.cc:1034] running on GCE node
08:17:05.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.509184 32302 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.512327 32302 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.513799 32302 hybrid_clock.cc:648] HybridClock initialized: now 1715069825513748 us; error 66 us; skew 500 ppm
08:17:05.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.514693 32302 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:05.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.522190 32302 webserver.cc:460] Webserver started at http://127.31.119.61:35347/ using document root <none> and password file <none>
08:17:05.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.523187 32302 fs_manager.cc:359] Metadata directory not provided
08:17:05.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.523450 32302 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:05.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.523878 32302 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.529110 32302 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data/instance:
08:17:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04a57fb2a3d74c62b1082cadc54ae366"
08:17:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:05 on dist-test-slave-gbgz"
08:17:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.530342 32302 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/wal/instance:
08:17:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04a57fb2a3d74c62b1082cadc54ae366"
08:17:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:05 on dist-test-slave-gbgz"
08:17:05.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.538146 32302 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
08:17:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.544342 32323 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.545704 32302 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.546020 32302 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/wal
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04a57fb2a3d74c62b1082cadc54ae366"
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:05 on dist-test-slave-gbgz"
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.546319 32302 fs_report.cc:383] FS layout report
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/wal
08:17:05.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/wal
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data/data
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:05.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.619467 32302 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.620985 32302 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.621433 32302 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.694936 32302 rpc_server.cc:307] RPC server started. Bound to: 127.31.119.61:43563
08:17:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.695019 32374 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.31.119.61:43563 every 8 connection(s)
08:17:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.697943 32302 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/data/info.pb
08:17:05.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.698621 32220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 32302
08:17:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.699105 32220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-1/wal/instance
08:17:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.702613 32220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/wal
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/logs
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data/info.pb
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:05.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.60:40891
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.60
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:05.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39843/jwks.json with env {}
08:17:05.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.705317 32375 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:17:05.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.718003 32375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
08:17:05.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.740032 32375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } has no permanent_uuid. Determining permanent_uuid...
08:17:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.759732 32375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } has no permanent_uuid. Determining permanent_uuid...
08:17:05.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:05.762298 32327 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:05.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:05.766963 32375 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:05.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.816596 32375 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 1
08:17:05.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:05.820878 32375 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:05.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.889410 32375 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 2
08:17:05.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:05.898991 32375 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:05.990411 32375 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 3
08:17:05.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:05.994478 32375 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.037856 32377 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.038411 32377 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:06.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.038877 32377 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.070842 32377 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.071134 32377 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.071386 32377 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.071635 32377 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:06.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.071831 32377 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.071966 32377 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39843/jwks.json
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.106806 32377 master_runner.cc:386] Master server non-default flags:
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/wal
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:06.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.60:40891
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39843/jwks.json
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data/info.pb
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.60
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:06.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/logs
08:17:06.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.108229 32377 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.110508 32377 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:06.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.131318 32387 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:06.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.137892 32375 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 4
08:17:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.141896 32375 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.316082 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } attempt: 7
08:17:06.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.328900 32300 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } has no permanent_uuid. Determining permanent_uuid...
08:17:06.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.334367 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.393101 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 1
08:17:06.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.397818 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.441397 32375 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 5
08:17:06.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.446390 32375 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.479501 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 2
08:17:06.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.485894 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.571406 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 3
08:17:06.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.576452 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.716902 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 4
08:17:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.723030 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:06.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.993831 32375 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 6
08:17:07.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.002282 32375 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:07.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.006507 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 5
08:17:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.012060 32300 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.31.119.60:40891: Network error: Client connection negotiation failed: client connection to 127.31.119.60:40891: connect: Connection refused (error 111)
08:17:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.131798 32390 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:06.132225 32388 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:07.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:06.135373 32377 server_base.cc:1034] running on GCE node
08:17:07.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.270727 32377 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:07.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.274360 32377 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:07.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.276244 32377 hybrid_clock.cc:648] HybridClock initialized: now 1715069827276202 us; error 38 us; skew 500 ppm
08:17:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.277459 32377 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:07.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.285250 32377 webserver.cc:460] Webserver started at http://127.31.119.60:39391/ using document root <none> and password file <none>
08:17:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.286386 32377 fs_manager.cc:359] Metadata directory not provided
08:17:07.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.286662 32377 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:07.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.287199 32377 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.292584 32377 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data/instance:
08:17:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a7767cb596cc48a9a8b53e9c5e78d354"
08:17:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:07 on dist-test-slave-gbgz"
08:17:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.293942 32377 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/wal/instance:
08:17:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a7767cb596cc48a9a8b53e9c5e78d354"
08:17:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:07 on dist-test-slave-gbgz"
08:17:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.301581 32377 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.003s
08:17:07.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.307848 32402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.308974 32377 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
08:17:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.309343 32377 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/wal
08:17:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a7767cb596cc48a9a8b53e9c5e78d354"
08:17:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:07 on dist-test-slave-gbgz"
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.309725 32377 fs_report.cc:383] FS layout report
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/wal
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/wal
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data/data
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:07.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.366060 32377 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:07.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.367753 32377 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:07.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.368304 32377 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.439857 32377 rpc_server.cc:307] RPC server started. Bound to: 127.31.119.60:40891
08:17:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.439954 32453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.31.119.60:40891 every 8 connection(s)
08:17:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.442693 32377 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/data/info.pb
08:17:07.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.449060 32454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:17:07.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.452713 32220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 32377
08:17:07.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.453157 32220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/master-2/wal/instance
08:17:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.461351 32454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
08:17:07.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.493922 32454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } has no permanent_uuid. Determining permanent_uuid...
08:17:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.506232 32454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } has no permanent_uuid. Determining permanent_uuid...
08:17:07.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.525175 32454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354: Bootstrap starting.
08:17:07.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.531908 32454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354: Neither blocks nor log segments found. Creating new log.
08:17:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.533777 32454 log.cc:824] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354: Log is configured to *not* fsync() on all Append() calls
08:17:07.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.538453 32454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354: No bootstrap required, opened a new log
08:17:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.560670 32454 raft_consensus.cc:348] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.561340 32454 raft_consensus.cc:374] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:17:07.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.561528 32454 raft_consensus.cc:729] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a7767cb596cc48a9a8b53e9c5e78d354, State: Initialized, Role: FOLLOWER
08:17:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.562250 32454 consensus_queue.cc:260] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.564792 32300 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 6
08:17:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.564963 32463 sys_catalog.cc:455] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.565958 32463 sys_catalog.cc:458] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: This master's current role is: FOLLOWER
08:17:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.567114 32454 sys_catalog.cc:564] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: configured and running, proceeding with master startup.
08:17:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.586831 32300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: Bootstrap starting.
08:17:07.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.592583 32300 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: Neither blocks nor log segments found. Creating new log.
08:17:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.594709 32300 log.cc:824] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: Log is configured to *not* fsync() on all Append() calls
08:17:07.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.606159 32300 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: No bootstrap required, opened a new log
08:17:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.608585 32475 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:07.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.608968 32475 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:07.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.624107 32408 tablet.cc:2369] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:17:07.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.634794 32300 raft_consensus.cc:348] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:07.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.636159 32300 raft_consensus.cc:374] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:17:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.636734 32300 raft_consensus.cc:729] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 34153164590348b8bd9c60db1d31c090, State: Initialized, Role: FOLLOWER
08:17:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.595364 32377 master_runner.cc:186] Error getting master registration for 127.31.119.62:45213: OK, instance_id { permanent_uuid: "34153164590348b8bd9c60db1d31c090" instance_seqno: 1715069823946614 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:17:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.637697 32377 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.638859 32300 consensus_queue.cc:260] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:07.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.642053 32478 sys_catalog.cc:455] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:07.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.643015 32478 sys_catalog.cc:458] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [sys.catalog]: This master's current role is: FOLLOWER
08:17:07.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.643512 32300 sys_catalog.cc:564] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [sys.catalog]: configured and running, proceeding with master startup.
08:17:07.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.654759 32254 tablet.cc:2369] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:17:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.677361 32489 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:07.678181 32489 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.674641 32232 master_runner.cc:186] Error getting master registration for 127.31.119.61:43563: OK, instance_id { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" instance_seqno: 1715069825674269 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:17:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:07.713577 32232 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.021320 32463 raft_consensus.cc:482] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.022002 32463 raft_consensus.cc:504] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.024645 32463 leader_election.cc:288] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 34153164590348b8bd9c60db1d31c090 (127.31.119.62:45213), 04a57fb2a3d74c62b1082cadc54ae366 (127.31.119.61:43563)
08:17:08.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.026340 32275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34153164590348b8bd9c60db1d31c090" is_pre_election: true
08:17:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.027218 32275 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a7767cb596cc48a9a8b53e9c5e78d354 in term 0.
08:17:08.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.028611 32404 leader_election.cc:302] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34153164590348b8bd9c60db1d31c090, a7767cb596cc48a9a8b53e9c5e78d354; no voters: 
08:17:08.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.029624 32463 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:17:08.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.030238 32463 raft_consensus.cc:482] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:17:08.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.030794 32463 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 0 FOLLOWER]: Advancing to term 1
08:17:08.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.043257 32463 raft_consensus.cc:504] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:08.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.047412 32463 leader_election.cc:288] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [CANDIDATE]: Term 1 election: Requested vote from peers 34153164590348b8bd9c60db1d31c090 (127.31.119.62:45213), 04a57fb2a3d74c62b1082cadc54ae366 (127.31.119.61:43563)
08:17:08.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.049746 32275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "34153164590348b8bd9c60db1d31c090"
08:17:08.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.050359 32275 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [term 0 FOLLOWER]: Advancing to term 1
08:17:08.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.062615 32375 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } attempt: 7
08:17:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.061956 32275 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a7767cb596cc48a9a8b53e9c5e78d354 in term 1.
08:17:08.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.069931 32404 leader_election.cc:302] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 34153164590348b8bd9c60db1d31c090, a7767cb596cc48a9a8b53e9c5e78d354; no voters: 
08:17:08.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.070752 32463 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 1 FOLLOWER]: Leader election won for term 1
08:17:08.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.082312 32463 raft_consensus.cc:686] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [term 1 LEADER]: Becoming Leader. State: Replica: a7767cb596cc48a9a8b53e9c5e78d354, State: Running, Role: LEADER
08:17:08.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.084687 32463 consensus_queue.cc:237] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:08.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.095840 32375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: Bootstrap starting.
08:17:08.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.103252 32375 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: Neither blocks nor log segments found. Creating new log.
08:17:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.109531 32375 log.cc:824] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: Log is configured to *not* fsync() on all Append() calls
08:17:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.112183 32492 sys_catalog.cc:455] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a7767cb596cc48a9a8b53e9c5e78d354. Latest consensus state: current_term: 1 leader_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.113035 32492 sys_catalog.cc:458] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: This master's current role is: LEADER
08:17:08.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.124397 32495 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
08:17:08.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.128206 32375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: No bootstrap required, opened a new log
08:17:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.131325 32495 catalog_manager.cc:1484] Initializing Kudu cluster ID...
08:17:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.163115 32375 raft_consensus.cc:348] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:08.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.164409 32375 raft_consensus.cc:374] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:17:08.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.164973 32375 raft_consensus.cc:729] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 04a57fb2a3d74c62b1082cadc54ae366, State: Initialized, Role: FOLLOWER
08:17:08.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.166290 32375 consensus_queue.cc:260] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } }
08:17:08.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.175041 32275 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [term 1 FOLLOWER]: Refusing update from remote peer a7767cb596cc48a9a8b53e9c5e78d354: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:17:08.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.178123 32492 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [LEADER]: Connected to new peer: Peer: permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:17:08.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.180287 32497 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.181124 32497 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [sys.catalog]: This master's current role is: FOLLOWER
08:17:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.182543 32375 sys_catalog.cc:564] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [sys.catalog]: configured and running, proceeding with master startup.
08:17:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.183496 32349 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04a57fb2a3d74c62b1082cadc54ae366"
08:17:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.183185 32350 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "04a57fb2a3d74c62b1082cadc54ae366" is_pre_election: true
08:17:08.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.183167 32348 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [term 0 FOLLOWER]: Advancing to term 1
08:17:08.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.204340 32348 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [term 1 FOLLOWER]: Refusing update from remote peer a7767cb596cc48a9a8b53e9c5e78d354: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:17:08.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.208851 32492 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [LEADER]: Connected to new peer: Peer: permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:17:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.247155 32498 sys_catalog.cc:455] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a7767cb596cc48a9a8b53e9c5e78d354. Latest consensus state: current_term: 1 leader_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.248283 32498 sys_catalog.cc:458] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [sys.catalog]: This master's current role is: FOLLOWER
08:17:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.253227 32496 mvcc.cc:205] Tried to move back new op lower bound from 7024926016174153728 to 7024926015870214144. Current Snapshot: MvccSnapshot[applied={T|T < 7024926016174153728}]
08:17:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.248421 32497 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a7767cb596cc48a9a8b53e9c5e78d354. Latest consensus state: current_term: 1 leader_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.255879 32497 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [sys.catalog]: This master's current role is: FOLLOWER
08:17:08.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.264376 32503 mvcc.cc:205] Tried to move back new op lower bound from 7024926016174153728 to 7024926015870214144. Current Snapshot: MvccSnapshot[applied={T|T < 7024926016174153728}]
08:17:08.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.281914 32504 mvcc.cc:205] Tried to move back new op lower bound from 7024926016174153728 to 7024926015870214144. Current Snapshot: MvccSnapshot[applied={T|T < 7024926016174153728}]
08:17:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.282799 32498 sys_catalog.cc:455] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.284466 32498 sys_catalog.cc:458] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090 [sys.catalog]: This master's current role is: FOLLOWER
08:17:08.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.289445 32495 catalog_manager.cc:1347] Generated new cluster ID: 5f4cc3b0e4fa4c8d824f8283df04ea95
08:17:08.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.289741 32495 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
08:17:08.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.297469 32463 sys_catalog.cc:455] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.298256 32463 sys_catalog.cc:458] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: This master's current role is: LEADER
08:17:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.298903 32463 sys_catalog.cc:455] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.299605 32463 sys_catalog.cc:458] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354 [sys.catalog]: This master's current role is: LEADER
08:17:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.300860 32497 sys_catalog.cc:455] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "34153164590348b8bd9c60db1d31c090" member_type: VOTER last_known_addr { host: "127.31.119.62" port: 45213 } } peers { permanent_uuid: "04a57fb2a3d74c62b1082cadc54ae366" member_type: VOTER last_known_addr { host: "127.31.119.61" port: 43563 } } peers { permanent_uuid: "a7767cb596cc48a9a8b53e9c5e78d354" member_type: VOTER last_known_addr { host: "127.31.119.60" port: 40891 } } }
08:17:08.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.301599 32497 sys_catalog.cc:458] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366 [sys.catalog]: This master's current role is: FOLLOWER
08:17:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.330050 32516 catalog_manager.cc:1259] Loaded cluster ID: 5f4cc3b0e4fa4c8d824f8283df04ea95
08:17:08.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.343892 32516 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: loading cluster ID for follower catalog manager: success
08:17:08.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.357615 32495 catalog_manager.cc:1370] Generated new certificate authority record
08:17:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:08.360008 32516 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: acquiring CA information for follower catalog manager: Not found: root CA entry not found
08:17:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:08.360375 32516 catalog_manager.cc:873] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
08:17:08.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.360149 32495 catalog_manager.cc:1504] Loading token signing keys...
08:17:08.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.402674 32495 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a7767cb596cc48a9a8b53e9c5e78d354: Generated new TSK 0
08:17:08.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.404186 32495 catalog_manager.cc:1514] Initializing in-progress tserver states...
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.440605 32220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/wal
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/logs
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data/info.pb
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.1:0
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.119.1
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.1
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:08.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:08.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:17:08.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.690940 32489 catalog_manager.cc:1259] Loaded cluster ID: 5f4cc3b0e4fa4c8d824f8283df04ea95
08:17:08.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.691251 32489 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: loading cluster ID for follower catalog manager: success
08:17:08.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.723443 32489 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: acquiring CA information for follower catalog manager: success
08:17:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:08.745507 32489 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 34153164590348b8bd9c60db1d31c090: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:17:09.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.033419 32518 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:09.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.034431 32518 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:09.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.035403 32518 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:09.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.092777 32518 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:09.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.094512 32518 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.119.1
08:17:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:09.157305 32518 tablet_server_runner.cc:78] Tablet server non-default flags:
08:17:09.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/wal
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.1:0
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data/info.pb
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.1
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.119.1
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/logs
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:09.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:17:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:09.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:09.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:09.166005 32518 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:09.168462 32518 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:09.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.212219 32525 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:09.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:09.369560 32516 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: acquiring CA information for follower catalog manager: success
08:17:09.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:09.375644 32516 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 04a57fb2a3d74c62b1082cadc54ae366: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:17:11.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.228061 32528 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:09.213402 32524 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:11.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:11.438552 32526 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2224 milliseconds
08:17:11.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.438668 32518 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:17:11.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.440439 32518 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:11.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.443699 32518 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:11.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.447324 32518 hybrid_clock.cc:648] HybridClock initialized: now 1715069831447221 us; error 99 us; skew 500 ppm
08:17:11.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.448472 32518 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:11.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.468950 32518 webserver.cc:460] Webserver started at http://127.31.119.1:44597/ using document root <none> and password file <none>
08:17:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.470216 32518 fs_manager.cc:359] Metadata directory not provided
08:17:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.470500 32518 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:11.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.471079 32518 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.478950 32518 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data/instance:
08:17:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85822f3aa6a945e8bf524f68b2d4e644"
08:17:11.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:11 on dist-test-slave-gbgz"
08:17:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.480542 32518 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/wal/instance:
08:17:11.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85822f3aa6a945e8bf524f68b2d4e644"
08:17:11.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:11 on dist-test-slave-gbgz"
08:17:11.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.490085 32518 fs_manager.cc:691] Time spent creating directory manager: real 0.009s	user 0.005s	sys 0.004s
08:17:11.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.497522 32534 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:11.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.498925 32518 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.000s
08:17:11.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.499363 32518 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/wal
08:17:11.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85822f3aa6a945e8bf524f68b2d4e644"
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:11 on dist-test-slave-gbgz"
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.499835 32518 fs_report.cc:383] FS layout report
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/wal
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/wal
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data/data
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:11.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.558104 32518 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:11.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.559760 32518 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:11.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.560230 32518 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:11.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.562881 32518 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:17:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.567255 32518 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:17:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.567518 32518 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.001s
08:17:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.567727 32518 ts_tablet_manager.cc:590] Registered 0 tablets
08:17:11.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.567873 32518 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:17:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.710019 32518 rpc_server.cc:307] RPC server started. Bound to: 127.31.119.1:42991
08:17:11.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.710089 32646 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.31.119.1:42991 every 8 connection(s)
08:17:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.713825 32518 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/data/info.pb
08:17:11.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.716492 32220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 32518
08:17:11.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.717017 32220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-0/wal/instance
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.733803 32220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/wal
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/logs
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data/info.pb
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.2:0
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.119.2
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.2
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:17:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.781304 32651 heartbeater.cc:347] Connected to a master server at 127.31.119.62:45213
08:17:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.781839 32651 heartbeater.cc:464] Registering TS with master...
08:17:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.781803 32648 heartbeater.cc:347] Connected to a master server at 127.31.119.61:43563
08:17:11.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.782521 32648 heartbeater.cc:464] Registering TS with master...
08:17:11.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.783905 32648 heartbeater.cc:511] Master 127.31.119.61:43563 requested a full tablet report, sending...
08:17:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.784696 32651 heartbeater.cc:511] Master 127.31.119.62:45213 requested a full tablet report, sending...
08:17:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.787381 32340 ts_manager.cc:194] Registered new tserver with Master: 85822f3aa6a945e8bf524f68b2d4e644 (127.31.119.1:42991)
08:17:11.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.787819 32265 ts_manager.cc:194] Registered new tserver with Master: 85822f3aa6a945e8bf524f68b2d4e644 (127.31.119.1:42991)
08:17:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.788908 32647 heartbeater.cc:347] Connected to a master server at 127.31.119.60:40891
08:17:11.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.789264 32647 heartbeater.cc:464] Registering TS with master...
08:17:11.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.790114 32647 heartbeater.cc:511] Master 127.31.119.60:40891 requested a full tablet report, sending...
08:17:11.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.792444 32419 ts_manager.cc:194] Registered new tserver with Master: 85822f3aa6a945e8bf524f68b2d4e644 (127.31.119.1:42991)
08:17:11.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:11.795125 32419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.119.1:39609
08:17:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.076694 32656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.077188 32656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:12.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.077621 32656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.109807 32656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.110630 32656 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.119.2
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:12.145756 32656 tablet_server_runner.cc:78] Tablet server non-default flags:
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/wal
08:17:12.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.2:0
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data/info.pb
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.2
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:12.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.119.2
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/logs
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:12.146970 32656 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:12.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:12.148732 32656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:12.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.166698 32665 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:12.798805 32647 heartbeater.cc:503] Master 127.31.119.60:40891 was elected leader, sending a full tablet report...
08:17:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.167515 32664 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:12.168308 32667 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:13.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:12.169106 32656 server_base.cc:1034] running on GCE node
08:17:13.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.295395 32656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:13.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.297871 32656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.299329 32656 hybrid_clock.cc:648] HybridClock initialized: now 1715069833299232 us; error 62 us; skew 500 ppm
08:17:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.300235 32656 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:13.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.306713 32656 webserver.cc:460] Webserver started at http://127.31.119.2:41201/ using document root <none> and password file <none>
08:17:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.307780 32656 fs_manager.cc:359] Metadata directory not provided
08:17:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.308073 32656 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:13.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.308616 32656 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.313561 32656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data/instance:
08:17:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61e5a4e8fb924a4da445b1fc690e52d1"
08:17:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:13 on dist-test-slave-gbgz"
08:17:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.314779 32656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/wal/instance:
08:17:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61e5a4e8fb924a4da445b1fc690e52d1"
08:17:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:13 on dist-test-slave-gbgz"
08:17:13.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.322249 32656 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.000s
08:17:13.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.328219 32676 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.329352 32656 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
08:17:13.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.329774 32656 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/wal
08:17:13.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61e5a4e8fb924a4da445b1fc690e52d1"
08:17:13.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:13 on dist-test-slave-gbgz"
08:17:13.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.330153 32656 fs_report.cc:383] FS layout report
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/wal
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/wal
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data/data
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:13.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.397027 32656 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:13.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.398653 32656 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:13.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.399195 32656 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:13.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.401875 32656 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:17:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.406347 32656 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:17:13.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.406596 32656 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:17:13.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.406914 32656 ts_tablet_manager.cc:590] Registered 0 tablets
08:17:13.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.407105 32656 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:17:13.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.572327 32656 rpc_server.cc:307] RPC server started. Bound to: 127.31.119.2:34087
08:17:13.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.572436   320 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.31.119.2:34087 every 8 connection(s)
08:17:13.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.575047 32656 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/data/info.pb
08:17:13.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.584307 32220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 32656
08:17:13.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.585152 32220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-1/wal/instance
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.615533 32220 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/wal
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/logs
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data/info.pb
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:13.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.3:0
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.119.3
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.3
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:13.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:17:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.640700   321 heartbeater.cc:347] Connected to a master server at 127.31.119.60:40891
08:17:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.641335   321 heartbeater.cc:464] Registering TS with master...
08:17:13.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.648257   321 heartbeater.cc:511] Master 127.31.119.60:40891 requested a full tablet report, sending...
08:17:13.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.652911 32419 ts_manager.cc:194] Registered new tserver with Master: 61e5a4e8fb924a4da445b1fc690e52d1 (127.31.119.2:34087)
08:17:13.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.654712 32419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.119.2:56765
08:17:13.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.662179   322 heartbeater.cc:347] Connected to a master server at 127.31.119.61:43563
08:17:13.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.662626   322 heartbeater.cc:464] Registering TS with master...
08:17:13.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.663396   322 heartbeater.cc:511] Master 127.31.119.61:43563 requested a full tablet report, sending...
08:17:13.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.666368 32340 ts_manager.cc:194] Registered new tserver with Master: 61e5a4e8fb924a4da445b1fc690e52d1 (127.31.119.2:34087)
08:17:13.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.707247   323 heartbeater.cc:347] Connected to a master server at 127.31.119.62:45213
08:17:13.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.707702   323 heartbeater.cc:464] Registering TS with master...
08:17:13.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.708406   323 heartbeater.cc:511] Master 127.31.119.62:45213 requested a full tablet report, sending...
08:17:13.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:13.711505 32265 ts_manager.cc:194] Registered new tserver with Master: 61e5a4e8fb924a4da445b1fc690e52d1 (127.31.119.2:34087)
08:17:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:13.975399   331 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:13.975893   331 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:13.976323   331 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:14.008486   331 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:14.009327   331 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.119.3
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:14.045059   331 tablet_server_runner.cc:78] Tablet server non-default flags:
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.119.20:43855
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/wal
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.119.3:0
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data/info.pb
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.119.3
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.119.62:45213,127.31.119.61:43563,127.31.119.60:40891
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.119.3
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/logs
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:14.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:14.046604   331 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:14.048790   331 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:14.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:14.066716   337 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:14.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:14.660853   321 heartbeater.cc:503] Master 127.31.119.60:40891 was elected leader, sending a full tablet report...
08:17:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:14.686422 32296 debug-util.cc:398] Leaking SignalData structure 0x7b08000a7ce0 after lost signal to thread 32233
08:17:14.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:14.687604 32296 debug-util.cc:398] Leaking SignalData structure 0x7b080009d260 after lost signal to thread 32299
08:17:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:14.068917   331 server_base.cc:1034] running on GCE node
08:17:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:14.068962   338 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:14.067489   340 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:15.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.224867   331 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:15.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.228286   331 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.230254   331 hybrid_clock.cc:648] HybridClock initialized: now 1715069835230151 us; error 92 us; skew 500 ppm
08:17:15.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.231585   331 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:15.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.240589   331 webserver.cc:460] Webserver started at http://127.31.119.3:40631/ using document root <none> and password file <none>
08:17:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.242029   331 fs_manager.cc:359] Metadata directory not provided
08:17:15.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.242372   331 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.243016   331 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.251945   331 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data/instance:
08:17:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "441b7ac5a7e94fb1a65b9da50d2eb3d9"
08:17:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:15 on dist-test-slave-gbgz"
08:17:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.253788   331 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/wal/instance:
08:17:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "441b7ac5a7e94fb1a65b9da50d2eb3d9"
08:17:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:15 on dist-test-slave-gbgz"
08:17:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.264827   331 fs_manager.cc:691] Time spent creating directory manager: real 0.010s	user 0.009s	sys 0.000s
08:17:15.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.273299   348 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.274863   331 fs_manager.cc:725] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.000s
08:17:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.275343   331 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/wal
08:17:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "441b7ac5a7e94fb1a65b9da50d2eb3d9"
08:17:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:15 on dist-test-slave-gbgz"
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.275842   331 fs_report.cc:383] FS layout report
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/wal
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/wal
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data/data
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:15.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.342936   331 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.344784   331 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:15.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.345352   331 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:15.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.348223   331 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
08:17:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.353111   331 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:17:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.353369   331 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:17:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.353626   331 ts_tablet_manager.cc:590] Registered 0 tablets
08:17:15.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.353809   331 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:17:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.543344   331 rpc_server.cc:307] RPC server started. Bound to: 127.31.119.3:32929
08:17:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.543467   460 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.31.119.3:32929 every 8 connection(s)
08:17:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.546630   331 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/data/info.pb
08:17:15.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.547220 32220 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 331
08:17:15.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.547768 32220 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster166710340951080975/ts-2/wal/instance
08:17:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.608520   467 heartbeater.cc:347] Connected to a master server at 127.31.119.62:45213
08:17:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.608506   462 heartbeater.cc:347] Connected to a master server at 127.31.119.61:43563
08:17:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.609115   467 heartbeater.cc:464] Registering TS with master...
08:17:15.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.609331   462 heartbeater.cc:464] Registering TS with master...
08:17:15.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.610805   462 heartbeater.cc:511] Master 127.31.119.61:43563 requested a full tablet report, sending...
08:17:15.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.610816   467 heartbeater.cc:511] Master 127.31.119.62:45213 requested a full tablet report, sending...
08:17:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.614472 32340 ts_manager.cc:194] Registered new tserver with Master: 441b7ac5a7e94fb1a65b9da50d2eb3d9 (127.31.119.3:32929)
08:17:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.614993   461 heartbeater.cc:347] Connected to a master server at 127.31.119.60:40891
08:17:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.615406   461 heartbeater.cc:464] Registering TS with master...
08:17:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.615399 32264 ts_manager.cc:194] Registered new tserver with Master: 441b7ac5a7e94fb1a65b9da50d2eb3d9 (127.31.119.3:32929)
08:17:15.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.616420   461 heartbeater.cc:511] Master 127.31.119.60:40891 requested a full tablet report, sending...
08:17:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.618541 32419 ts_manager.cc:194] Registered new tserver with Master: 441b7ac5a7e94fb1a65b9da50d2eb3d9 (127.31.119.3:32929)
08:17:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.619928 32419 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.31.119.3:41593
08:17:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:15.639642 32220 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
08:17:15.655 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:17:15.657 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

08:17:15.670 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.31.119.62"
      port: 45213
    }
    bound_http_address {
      host: "127.31.119.62"
      port: 40975
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.31.119.61"
      port: 43563
    }
    bound_http_address {
      host: "127.31.119.61"
      port: 35347
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.31.119.60"
      port: 40891
    }
    bound_http_address {
      host: "127.31.119.60"
      port: 39391
    }
  }
}

08:17:15.677 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

08:17:15.680 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.31.119.1"
      port: 42991
    }
    bound_http_address {
      host: "127.31.119.1"
      port: 44597
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.31.119.2"
      port: 34087
    }
    bound_http_address {
      host: "127.31.119.2"
      port: 41201
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.31.119.3"
      port: 32929
    }
    bound_http_address {
      host: "127.31.119.3"
      port: 40631
    }
  }
}

08:17:15.681 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:17:15.939 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
08:17:16.093 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskN8qzGC/test-tmp
08:17:16.094 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:17:16.095 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskN8qzGC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:17:16.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 0
  enable_kerberos: true
  cluster_root: "/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091"
  extra_master_flags: "--authn_token_validity_seconds=10"
  hms_mode: NONE
  mini_kdc_options {
    ticket_lifetime: "10s"
    renew_lifetime: "20s"
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

08:17:16.443 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:16.444 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}

08:17:16.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:16.449131   473 mini_oidc.cc:152] Starting JWKS server
08:17:16.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:16.454044   473 webserver.cc:460] Webserver started at http://127.0.0.1:40693/ using document root <none> and password file <none>
08:17:16.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:16.454978   473 mini_oidc.cc:189] Starting OIDC Discovery server
08:17:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:16.457589   473 webserver.cc:460] Webserver started at http://127.0.0.1:40295/ using document root <none> and password file <none>
08:17:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:17:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/principal' for realm 'KRBTEST.COM',
08:17:16.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:17:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:16 dist-test-slave-gbgz krb5kdc[485](info): setting up network...
08:17:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:17:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:16 dist-test-slave-gbgz krb5kdc[485](info): set up 2 sockets
08:17:16.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:16 dist-test-slave-gbgz krb5kdc[485](info): commencing operation
08:17:16.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:17:16.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:16.624435   461 heartbeater.cc:503] Master 127.31.119.60:40891 was elected leader, sending a full tablet report...
08:17:17.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:17.537436   473 mini_kdc.cc:121] Time spent starting KDC: real 1.030s	user 0.003s	sys 0.020s
08:17:17.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:17:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:17:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:17:17.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:17:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:17:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:17:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:17:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:17:17.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:17:17.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:17:17.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/test-user.keytab.
08:17:17.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/test-user.keytab.
08:17:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:17 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069837, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
08:17:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-07T08:17:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
08:17:17.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-07T08:17:17Z Disabled control of system clock
08:17:17.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.118.126@KRBTEST.COM; defaulting to no policy
08:17:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:17:17.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.118.126@KRBTEST.COM" created.
08:17:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:17:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.118.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.126.keytab.
08:17:17.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.118.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.126.keytab.
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:17.758108   473 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/logs
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/info.pb
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:17.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.126:35389
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.126
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.126.keytab
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.126
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:17.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:17.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:17:18.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.073421   498 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.073987   498 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.074615   498 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.107187   498 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:17:18.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.107499   498 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.107816   498 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.108021   498 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.108203   498 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.108392   498 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.108533   498 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40693/jwks.json
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:18.146775   498 master_runner.cc:386] Master server non-default flags:
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.126.keytab
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.126
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.126:35389
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:18.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/info.pb
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.126
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/logs
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:18.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:18.148082   498 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:18.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:18.149168   498 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:18.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.167482   504 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.168112   505 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:19.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:18.168743   507 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:19.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:18.168872   498 server_base.cc:1034] running on GCE node
08:17:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.344405   498 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:19.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.348030   498 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:19.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.349539   498 hybrid_clock.cc:648] HybridClock initialized: now 1715069839349477 us; error 65 us; skew 500 ppm
08:17:19.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.350404   498 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:19 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069839, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:19.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.356969   498 init.cc:377] Logged in from keytab as kudu/127.0.118.126@KRBTEST.COM (short username kudu)
08:17:19.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.362695   498 webserver.cc:460] Webserver started at http://127.0.118.126:38797/ using document root <none> and password file <none>
08:17:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.363823   498 fs_manager.cc:359] Metadata directory not provided
08:17:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.364110   498 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.364586   498 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.370455   498 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/instance:
08:17:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ded8178b9de45ff8f6d98015be4e05d"
08:17:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:19 on dist-test-slave-gbgz"
08:17:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.371858   498 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal/instance:
08:17:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ded8178b9de45ff8f6d98015be4e05d"
08:17:19.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:19 on dist-test-slave-gbgz"
08:17:19.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.380066   498 fs_manager.cc:691] Time spent creating directory manager: real 0.008s	user 0.006s	sys 0.001s
08:17:19.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.386202   516 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.387449   498 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.387773   498 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ded8178b9de45ff8f6d98015be4e05d"
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:19 on dist-test-slave-gbgz"
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.388101   498 fs_report.cc:383] FS layout report
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/data
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:19.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.485275   498 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:19.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.491178   498 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:19.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.491705   498 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.569489   498 rpc_server.cc:307] RPC server started. Bound to: 127.0.118.126:35389
08:17:19.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.569649   567 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.118.126:35389 every 8 connection(s)
08:17:19.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.572484   498 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/info.pb
08:17:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.578574   473 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 498
08:17:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.579061   473 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal/instance
08:17:19.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.579576   568 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:17:19.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.593250   568 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } has no permanent_uuid. Determining permanent_uuid...
08:17:19.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.118.125@KRBTEST.COM; defaulting to no policy
08:17:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:17:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.118.125@KRBTEST.COM" created.
08:17:19.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:19 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069839, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.126@KRBTEST.COM for kudu/127.0.118.126@KRBTEST.COM
08:17:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:17:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.118.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.125.keytab.
08:17:19.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.118.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.125.keytab.
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.637394   568 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } has no permanent_uuid. Determining permanent_uuid...
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.638029   473 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/logs
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/info.pb
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.125:46525
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.125
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.125.keytab
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.125
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:17:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:17:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.642778   519 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:19.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.647342   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.125:46525: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:19.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.696974   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } attempt: 1
08:17:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.702167   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.125:46525: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.770674   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } attempt: 2
08:17:19.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.775471   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.125:46525: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:19.866914   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } attempt: 3
08:17:19.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.871472   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.125:46525: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.977690   574 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.978404   574 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:19.979151   574 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:20.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.011432   574 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:17:20.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.011739   574 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.012157   574 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.012472   574 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.012780   574 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.013092   574 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:20.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.013309   574 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40693/jwks.json
08:17:20.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:20.014940   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } attempt: 4
08:17:20.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.018997   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.125:46525: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:20.052029   574 master_runner.cc:386] Master server non-default flags:
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.125.keytab
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.125
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.125:46525
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:20.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/info.pb
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.125
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/logs
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:20.053207   574 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:20.054407   574 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:20.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.071221   581 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:20.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:20.318437   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } attempt: 5
08:17:20.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.324436   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.125:46525: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:20.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:20.871934   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } attempt: 6
08:17:20.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.879675   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.125:46525: Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:21.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:20.072835   574 server_base.cc:1034] running on GCE node
08:17:21.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.072368   582 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:21.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:20.071436   584 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:21.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.227319   574 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.230968   574 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:21.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.232484   574 hybrid_clock.cc:648] HybridClock initialized: now 1715069841232454 us; error 47 us; skew 500 ppm
08:17:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.233295   574 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:21.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:21 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069841, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:21.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.239607   574 init.cc:377] Logged in from keytab as kudu/127.0.118.125@KRBTEST.COM (short username kudu)
08:17:21.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.245296   574 webserver.cc:460] Webserver started at http://127.0.118.125:37027/ using document root <none> and password file <none>
08:17:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.246366   574 fs_manager.cc:359] Metadata directory not provided
08:17:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.246595   574 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.247133   574 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:21.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.252424   574 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/instance:
08:17:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a70fabb270b4dbcbb09b9e1cac7475d"
08:17:21.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:21 on dist-test-slave-gbgz"
08:17:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.253764   574 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal/instance:
08:17:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a70fabb270b4dbcbb09b9e1cac7475d"
08:17:21.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:21 on dist-test-slave-gbgz"
08:17:21.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.261509   574 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.000s
08:17:21.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.267566   594 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.268744   574 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.000s
08:17:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.269109   574 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a70fabb270b4dbcbb09b9e1cac7475d"
08:17:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:21 on dist-test-slave-gbgz"
08:17:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.269435   574 fs_report.cc:383] FS layout report
08:17:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:21.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/data
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:21.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.339170   574 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.345410   574 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:21.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.345854   574 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.418872   574 rpc_server.cc:307] RPC server started. Bound to: 127.0.118.125:46525
08:17:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.418968   645 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.118.125:46525 every 8 connection(s)
08:17:21.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.421957   574 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/info.pb
08:17:21.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.427332   646 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:17:21.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.430704   473 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 574
08:17:21.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.431143   473 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal/instance
08:17:21.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.439505   646 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } has no permanent_uuid. Determining permanent_uuid...
08:17:21.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.0.118.124@KRBTEST.COM; defaulting to no policy
08:17:21.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:17:21.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.0.118.124@KRBTEST.COM" created.
08:17:21.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:21 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069841, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.125@KRBTEST.COM for kudu/127.0.118.126@KRBTEST.COM
08:17:21.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.479457   646 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } has no permanent_uuid. Determining permanent_uuid...
08:17:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:17:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.118.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.124.keytab.
08:17:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.0.118.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.124.keytab.
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.488927   473 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/logs
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/info.pb
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:21.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.124:35773
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.124
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.124.keytab
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.124
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:17:21.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:21 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069841, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.125@KRBTEST.COM for kudu/127.0.118.125@KRBTEST.COM
08:17:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.506307   646 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } has no permanent_uuid. Determining permanent_uuid...
08:17:21.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.509025   596 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:21.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.513123   646 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:21.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.562638   646 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 1
08:17:21.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.567723   646 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.636214   646 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 2
08:17:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.641450   646 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.732972   646 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 3
08:17:21.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.737257   646 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.835984   653 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:21.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.836542   653 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:21.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.837128   653 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.869818   653 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:17:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.870059   653 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.870322   653 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.870522   653 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:21.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.870713   653 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.870910   653 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.871055   653 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40693/jwks.json
08:17:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.880669   646 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 4
08:17:21.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.884896   646 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.909737   653 master_runner.cc:386] Master server non-default flags:
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.124.keytab
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.124
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.124:35773
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:21.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/info.pb
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.124
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/logs
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.910895   653 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.911957   653 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.928579   662 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:21.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.940125   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } attempt: 7
08:17:21.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:21 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069839, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.126@KRBTEST.COM for kudu/127.0.118.125@KRBTEST.COM
08:17:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.973138   568 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } has no permanent_uuid. Determining permanent_uuid...
08:17:21.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.981148   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:22.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:22.039629   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 1
08:17:22.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:22.044015   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:22.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:22.125717   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 2
08:17:22.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:22.130218   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:22.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:22.184345   646 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 5
08:17:22.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:22.189508   646 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:22.215929   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 3
08:17:22.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:22.223066   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:22.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:22.363607   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 4
08:17:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:22.368875   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:22.652585   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 5
08:17:22.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:22.658048   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:22.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:22.736950   646 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 6
08:17:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:22.746346   646 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:23.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.929232   661 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:21.930270   653 server_base.cc:1034] running on GCE node
08:17:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:21.929783   664 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.113605   653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:23.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.116932   653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:23.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.118453   653 hybrid_clock.cc:648] HybridClock initialized: now 1715069843118404 us; error 63 us; skew 500 ppm
08:17:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.119478   653 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:23.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069843, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:23.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.125908   653 init.cc:377] Logged in from keytab as kudu/127.0.118.124@KRBTEST.COM (short username kudu)
08:17:23.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.131601   653 webserver.cc:460] Webserver started at http://127.0.118.124:39997/ using document root <none> and password file <none>
08:17:23.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.132504   653 fs_manager.cc:359] Metadata directory not provided
08:17:23.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.132701   653 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:23.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.133077   653 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
08:17:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.138073   653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/instance:
08:17:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "517bd80235974c17827d8c97cb89bece"
08:17:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:23 on dist-test-slave-gbgz"
08:17:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.139254   653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal/instance:
08:17:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "517bd80235974c17827d8c97cb89bece"
08:17:23.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:23 on dist-test-slave-gbgz"
08:17:23.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.147010   653 fs_manager.cc:691] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.005s
08:17:23.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.152765   678 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.153961   653 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.002s
08:17:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.154289   653 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "517bd80235974c17827d8c97cb89bece"
08:17:23.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:23 on dist-test-slave-gbgz"
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.154704   653 fs_report.cc:383] FS layout report
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/data
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:23.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:23.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.210485   568 consensus_peers.cc:654] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } attempt: 6
08:17:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:23.217002   568 consensus_peers.cc:644] Error getting permanent uuid from config peer 127.0.118.124:35773: Network error: Client connection negotiation failed: client connection to 127.0.118.124:35773: connect: Connection refused (error 111)
08:17:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.227138   653 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.232846   653 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.233273   653 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.305186   653 rpc_server.cc:307] RPC server started. Bound to: 127.0.118.124:35773
08:17:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.305277   731 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.118.124:35773 every 8 connection(s)
08:17:23.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.307998   653 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/info.pb
08:17:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.310004   473 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 653
08:17:23.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.310496   473 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal/instance
08:17:23.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.316314   732 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:17:23.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.337221   732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } has no permanent_uuid. Determining permanent_uuid...
08:17:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069843, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for kudu/127.0.118.126@KRBTEST.COM
08:17:23.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069837, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.118.124@KRBTEST.COM
08:17:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.375459   732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } has no permanent_uuid. Determining permanent_uuid...
08:17:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069843, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for kudu/127.0.118.125@KRBTEST.COM
08:17:23.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069837, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.118.125@KRBTEST.COM
08:17:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.400511   732 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } has no permanent_uuid. Determining permanent_uuid...
08:17:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069843, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for kudu/127.0.118.124@KRBTEST.COM
08:17:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069837, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.118.126@KRBTEST.COM
08:17:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.431352   732 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Bootstrap starting.
08:17:23.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.438127   473 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
08:17:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.440341   732 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Neither blocks nor log segments found. Creating new log.
08:17:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.442953   732 log.cc:824] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Log is configured to *not* fsync() on all Append() calls
08:17:23.449 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:23.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}

08:17:23.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.459681   732 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: No bootstrap required, opened a new log
08:17:23.468 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
  env_vars {
    key: "KRB5CCNAME"
    value: "/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc"
  }
  env_vars {
    key: "KRB5_CONFIG"
    value: "/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5.conf"
  }
  env_vars {
    key: "KRB5_KDC_PROFILE"
    value: "/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kdc.conf"
  }
  env_vars {
    key: "KUDU_ENABLE_KRB5_REALM_FIX"
    value: "yes"
  }
}

08:17:23.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}

08:17:23.472 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.0.118.126"
      port: 35389
    }
    bound_http_address {
      host: "127.0.118.126"
      port: 38797
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.0.118.125"
      port: 46525
    }
    bound_http_address {
      host: "127.0.118.125"
      port: 37027
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.0.118.124"
      port: 35773
    }
    bound_http_address {
      host: "127.0.118.124"
      port: 39997
    }
  }
}

08:17:23.474 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}

08:17:23.474 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:23.474 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:17:23.476 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

08:17:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.497290   732 raft_consensus.cc:348] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.498338   732 raft_consensus.cc:374] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:17:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.498693   732 raft_consensus.cc:729] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 517bd80235974c17827d8c97cb89bece, State: Initialized, Role: FOLLOWER
08:17:23.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.499788   732 consensus_queue.cc:260] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.503111   743 sys_catalog.cc:455] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } } }
08:17:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.504070   743 sys_catalog.cc:458] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [sys.catalog]: This master's current role is: FOLLOWER
08:17:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.505038   732 sys_catalog.cc:564] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [sys.catalog]: configured and running, proceeding with master startup.
08:17:23.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069843, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
08:17:23.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:23.524 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc
08:17:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:23.533547   754 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:23.534 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:17:23.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:23.534364   754 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:23.535 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc
08:17:23.539 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:17:23.544 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:17:23.547 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:17:23.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

08:17:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.529518   653 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: OK, instance_id { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" instance_seqno: 1715069839548513 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:17:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.576022   653 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:23 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069843, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:23.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
08:17:23.632 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:23.632 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc
08:17:23.634 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:17:23.634 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.118.124:35773
08:17:23.636 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

08:17:23.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.637501   473 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 653
08:17:23.665 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:23.666 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.118.125:46525
08:17:23.666 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

08:17:23.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.667379   473 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 574
08:17:23.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:23.694 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.0.118.126:35389
08:17:23.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

08:17:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.694953   473 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 498
08:17:23.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:23.725 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.118.124:35773
08:17:23.727 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 2
  }
}

08:17:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:23.730101   473 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data
08:17:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/logs
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/info.pb
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.124:35773
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.124
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39997
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.124.keytab
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.124
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:17:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.068552   758 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.069183   758 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:24.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.069809   758 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.101748   758 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:17:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.101998   758 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.102295   758 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.102500   758 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.102772   758 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.103026   758 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:24.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.103178   758 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40693/jwks.json
08:17:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:24.143918   758 master_runner.cc:386] Master server non-default flags:
08:17:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:24.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.124.keytab
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.124
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.124:35773
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:24.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/info.pb
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.124
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39997
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/logs
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:24.145298   758 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:24.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:24.146301   758 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:24.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.164238   765 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.164925   766 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:24.165756   768 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:25.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:24.168399   758 server_base.cc:1034] running on GCE node
08:17:25.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.364377   758 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:25.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.367458   758 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:25.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.368875   758 hybrid_clock.cc:648] HybridClock initialized: now 1715069845368836 us; error 53 us; skew 500 ppm
08:17:25.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.369694   758 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:25.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:25 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069845, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:25.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.375607   758 init.cc:377] Logged in from keytab as kudu/127.0.118.124@KRBTEST.COM (short username kudu)
08:17:25.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.380950   758 webserver.cc:460] Webserver started at http://127.0.118.124:39997/ using document root <none> and password file <none>
08:17:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.381999   758 fs_manager.cc:359] Metadata directory not provided
08:17:25.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.382270   758 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.390249   758 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.005s	sys 0.000s
08:17:25.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.394711   778 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:25.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.395870   758 fs_manager.cc:725] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.000s
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.396250   758 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "517bd80235974c17827d8c97cb89bece"
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:23 on dist-test-slave-gbgz"
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.399176   758 fs_report.cc:383] FS layout report
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/wal
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/data
08:17:25.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:25.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:25.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.464790   758 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.470968   758 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.471537   758 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.544845   758 rpc_server.cc:307] RPC server started. Bound to: 127.0.118.124:35773
08:17:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.545064   829 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.118.124:35773 every 8 connection(s)
08:17:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.547917   758 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-2/data/info.pb
08:17:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.552434   473 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 758
08:17:25.553 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:25.553 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.118.125:46525
08:17:25.553 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 1
  }
}

08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.554982   473 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/logs
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/info.pb
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:25.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.125:46525
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.125
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37027
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.125.keytab
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.125
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:17:25.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:25.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:17:25.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.562013   830 sys_catalog.cc:263] Verifying existing consensus state
08:17:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.572925   830 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Bootstrap starting.
08:17:25.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.619324   830 log.cc:824] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Log is configured to *not* fsync() on all Append() calls
08:17:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.625280   830 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:17:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.626245   830 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Bootstrap complete.
08:17:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.662670   830 raft_consensus.cc:348] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.666232   830 raft_consensus.cc:374] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:17:25.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.666580   830 raft_consensus.cc:729] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 517bd80235974c17827d8c97cb89bece, State: Initialized, Role: FOLLOWER
08:17:25.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.667614   830 consensus_queue.cc:260] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:25.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.670477   834 sys_catalog.cc:455] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } } }
08:17:25.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.671125   834 sys_catalog.cc:458] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [sys.catalog]: This master's current role is: FOLLOWER
08:17:25.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.672430   830 sys_catalog.cc:564] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [sys.catalog]: configured and running, proceeding with master startup.
08:17:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:25.706318   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:25.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:25.706694   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:25.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:25.726667   784 tablet.cc:2369] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:17:25.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.709848   758 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111), 
08:17:25.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:25.753281   758 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:25.965356   832 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:25.966002   832 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:25.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:25.966773   832 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.000249   832 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:17:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.000489   832 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:26.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.000759   832 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.000995   832 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.001206   832 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.001441   832 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.001614   832 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40693/jwks.json
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.042992   832 master_runner.cc:386] Master server non-default flags:
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.125.keytab
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.125
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.125:46525
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/info.pb
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.125
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37027
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/logs
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:26.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:26.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.044445   832 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:26.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.045945   832 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.065078   854 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:26.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.137482   834 raft_consensus.cc:482] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:26.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.138295   834 raft_consensus.cc:504] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:26.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.145886   779 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:26.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.154062   834 leader_election.cc:288] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389), 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525)
08:17:26.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.171379   779 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389): Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:26.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.176162   781 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525): Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.176885   781 leader_election.cc:302] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 517bd80235974c17827d8c97cb89bece; no voters: 1ded8178b9de45ff8f6d98015be4e05d, 4a70fabb270b4dbcbb09b9e1cac7475d
08:17:26.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:26.178313   834 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:26.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.302536 32450 debug-util.cc:398] Leaking SignalData structure 0x7b08000bf820 after lost signal to thread 32382
08:17:26.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.306054 32450 debug-util.cc:398] Leaking SignalData structure 0x7b0800091800 after lost signal to thread 32453
08:17:26.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.710777   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:26.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.711133   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.069123   853 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:26.076102   856 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:27.534868   855 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1471 milliseconds
08:17:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.534898   832 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:17:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.541968   832 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:27.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.560223   832 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.562001   832 hybrid_clock.cc:648] HybridClock initialized: now 1715069847561913 us; error 82 us; skew 500 ppm
08:17:27.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.564442   832 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:27.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:27 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069847, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:27.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.578836   832 init.cc:377] Logged in from keytab as kudu/127.0.118.125@KRBTEST.COM (short username kudu)
08:17:27.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.603631   832 webserver.cc:460] Webserver started at http://127.0.118.125:37027/ using document root <none> and password file <none>
08:17:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.604781   832 fs_manager.cc:359] Metadata directory not provided
08:17:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.605057   832 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:27.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.625802   832 fs_manager.cc:709] Time spent opening directory manager: real 0.017s	user 0.007s	sys 0.008s
08:17:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.644326   867 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:27.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.648928   832 fs_manager.cc:725] Time spent opening block manager: real 0.020s	user 0.006s	sys 0.008s
08:17:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.649699   832 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a70fabb270b4dbcbb09b9e1cac7475d"
08:17:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:21 on dist-test-slave-gbgz"
08:17:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.655081   832 fs_report.cc:383] FS layout report
08:17:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/data
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:27.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:27.715384   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:27.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:27.715724   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.796809   872 raft_consensus.cc:482] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.797235   832 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.797950   872 raft_consensus.cc:504] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.806105   832 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.807034   832 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:27.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.815266   872 leader_election.cc:288] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389), 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525)
08:17:27.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:27.827293   781 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525): Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:27.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:27.828857   779 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389): Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:27.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.829726   779 leader_election.cc:302] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 517bd80235974c17827d8c97cb89bece; no voters: 1ded8178b9de45ff8f6d98015be4e05d, 4a70fabb270b4dbcbb09b9e1cac7475d
08:17:27.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.831063   872 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.958045   832 rpc_server.cc:307] RPC server started. Bound to: 127.0.118.125:46525
08:17:27.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.958199   921 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.118.125:46525 every 8 connection(s)
08:17:27.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.962379   832 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/data/info.pb
08:17:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.970752   473 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 832
08:17:27.972 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:27.973 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.0.118.126:35389
08:17:27.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 0
  }
}

08:17:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.976154   473 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu
08:17:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data
08:17:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:17:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/logs
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/info.pb
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:17:27.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.126:35389
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.126
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38797
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.126.keytab
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.126
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:27.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:27.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:27.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:27.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:27.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:17:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.990520   922 sys_catalog.cc:263] Verifying existing consensus state
08:17:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240507 08:17:27.991904   922 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:17:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:27.992602   832 master.cc:537] Master@127.0.118.125:46525 shutting down...
08:17:28.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:28.104503   832 master.cc:552] Master@127.0.118.125:46525 shutdown complete.
08:17:28.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:17:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.450450   924 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:17:28.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.451547   924 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:17:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.452713   924 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:17:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.509855   924 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:17:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.510196   924 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:17:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.510656   924 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:17:28.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.511009   924 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:17:28.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.511355   924 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:17:28.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.511688   924 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:17:28.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.511936   924 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40693/jwks.json
08:17:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:28.576284   924 master_runner.cc:386] Master server non-default flags:
08:17:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:17:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.118.84:36965
08:17:28.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.118.126:35389,127.0.118.125:46525,127.0.118.124:35773
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/krb5kdc/kudu_127.0.118.126.keytab
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.0.118.126
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.118.126:35389
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40693/jwks.json
08:17:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/info.pb
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.118.126
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38797
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/logs
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 259fcc2b501077402e3d0e0d474a7ca475b70b5f
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 07 May 2024 07:43:19 UTC on 1a15035a6830
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 176
08:17:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:17:28.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:28.587769   924 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:17:28.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:28.589280   924 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:17:28.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.648306   931 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.719938   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.720546   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:29.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:29.511129   937 raft_consensus.cc:482] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:29.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:29.512514   937 raft_consensus.cc:504] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:29.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:29.527706   937 leader_election.cc:288] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389), 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525)
08:17:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:29.557186   781 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525): Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:29.558146   779 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389): Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:29.558744   779 leader_election.cc:302] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 517bd80235974c17827d8c97cb89bece; no voters: 1ded8178b9de45ff8f6d98015be4e05d, 4a70fabb270b4dbcbb09b9e1cac7475d
08:17:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:29.559679   937 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:29.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:29.726574   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:29.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:29.727039   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.019126   929 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 924
08:17:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.167143   929 kernel_stack_watchdog.cc:198] Thread 924 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 357ms:
08:17:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:17:30.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:17:30.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:17:30.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:17:30.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:28.650636   930 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:30.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.169076   924 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.510s	user 0.002s	sys 0.000s
08:17:30.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.169440   924 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.510s	user 0.003s	sys 0.000s
08:17:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.173590   932 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1510 milliseconds
08:17:30.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.176067   933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:17:30.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.176152   924 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:17:30.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.177273   924 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:17:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.179745   924 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:17:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.181085   924 hybrid_clock.cc:648] HybridClock initialized: now 1715069850181053 us; error 44 us; skew 500 ppm
08:17:30.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.181936   924 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:17:30.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:30 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069850, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:30.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.187940   924 init.cc:377] Logged in from keytab as kudu/127.0.118.126@KRBTEST.COM (short username kudu)
08:17:30.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.193037   924 webserver.cc:460] Webserver started at http://127.0.118.126:38797/ using document root <none> and password file <none>
08:17:30.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.193925   924 fs_manager.cc:359] Metadata directory not provided
08:17:30.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.194136   924 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:17:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.202272   924 fs_manager.cc:709] Time spent opening directory manager: real 0.005s	user 0.002s	sys 0.004s
08:17:30.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.206476   946 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:17:30.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.207602   924 fs_manager.cc:725] Time spent opening block manager: real 0.003s	user 0.003s	sys 0.000s
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.207921   924 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data,/tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ded8178b9de45ff8f6d98015be4e05d"
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2024-05-07 08:17:19 on dist-test-slave-gbgz"
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.210189   924 fs_report.cc:383] FS layout report
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/data
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:17:30.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:17:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:17:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:17:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:17:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:17:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:17:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:17:30.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.274148   924 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:17:30.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.279996   924 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:17:30.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.280419   924 kserver.cc:163] Server-wide thread pool size limit: 3276
08:17:30.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.354221   924 rpc_server.cc:307] RPC server started. Bound to: 127.0.118.126:35389
08:17:30.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.354467   997 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.118.126:35389 every 8 connection(s)
08:17:30.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.357152   924 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/data/info.pb
08:17:30.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.358992   473 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu as pid 924
08:17:30.360 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:17:30.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.372110   998 sys_catalog.cc:263] Verifying existing consensus state
08:17:30.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20240507 08:17:30.373217   998 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:17:30.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.373930   924 master.cc:537] Master@127.0.118.126:35389 shutting down...
08:17:30.424 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.118.124:35773(127.0.118.124:35773)] connecting to peer
08:17:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.464221   924 master.cc:552] Master@127.0.118.126:35389 shutdown complete.
08:17:30.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskN8qzGC/test-tmp/mini-kudu-cluster8583619543717751091/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:17:30.486 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:30.488 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:30.556 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.556 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.559 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.559 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.0.118.124:35773(127.0.118.124:35773)] Successfully connected to peer
08:17:30.614 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.614 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.616 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.633 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:17:30.642 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.0.118.124
08:17:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.730978   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:30.731642   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.757225   758 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111), 
08:17:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:30.757615   758 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:30.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:30 dist-test-slave-gbgz krb5kdc[485](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069843, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.0.118.124@KRBTEST.COM
08:17:30.866 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x94269427, L:/127.0.0.1:46832 - R:/127.0.118.124:35773] using SASL/GSSAPI
08:17:30.882 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:30.888 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:30.889 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:30.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.891 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.894 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.895 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.896 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:30.906 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:30.914 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:30.914 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.914 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.916 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.916 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.916 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.917 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.918 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:30.945 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:30.947 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:30.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.952 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:30.966 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:30.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:30.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.974 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.974 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:30.986 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:30.988 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:30.991 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.992 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.993 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:30.993 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.993 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.993 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:30.994 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:31.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:31.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:31.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.051 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.051 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.052 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.052 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:31.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:31.088 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:31.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.091 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.093 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.093 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.094 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.094 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:31.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:31.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:31.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:31.446 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:31.449 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:31.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.451 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.452 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.452 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.452 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.453 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:31.686 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:31.688 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:31.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.691 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.691 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.691 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:31.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:31.734633   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:31.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:31.734910   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:31.806 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:31.809 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:31.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:31.811 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.811 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.811 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:31.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:32.014647  1006 raft_consensus.cc:482] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:32.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:32.015040  1006 raft_consensus.cc:504] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:32.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:32.016690  1006 leader_election.cc:288] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389), 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525)
08:17:32.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:32.018829   779 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111) [suppressed 5 similar messages]
08:17:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:32.021780   781 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525): Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:32.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:32.022578   779 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389): Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:32.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:32.023371   779 leader_election.cc:302] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 517bd80235974c17827d8c97cb89bece; no voters: 1ded8178b9de45ff8f6d98015be4e05d, 4a70fabb270b4dbcbb09b9e1cac7475d
08:17:32.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:32.024770  1006 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:32.737871   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:32.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:32.738154   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:33.741310   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:33.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:33.741586   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:34.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:34.510105  1015 raft_consensus.cc:482] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:34.510885  1015 raft_consensus.cc:504] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:34.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:34.513161  1015 leader_election.cc:288] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389), 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525)
08:17:34.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:34.518829   781 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525): Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:34.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:34.519452   779 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389): Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:34.519783   779 leader_election.cc:302] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 517bd80235974c17827d8c97cb89bece; no voters: 1ded8178b9de45ff8f6d98015be4e05d, 4a70fabb270b4dbcbb09b9e1cac7475d
08:17:34.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:34.520357  1015 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:34.686 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:34.687 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:34.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:34.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:34.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:34.690 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:34.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:34.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:34.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:34.745659   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:34.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:34.746034   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:35.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:35 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069855, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:35.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:35.379397   774 init.cc:295] Successfully reacquired a new kerberos TGT
08:17:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:35.749837   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:35.750115   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:35.760704   758 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111), 
08:17:35.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:35.761155   758 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:36.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:36.753566   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:36.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:36.753847   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:36.966 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:36.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:36.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:36.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:36.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:36.971 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:36.971 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:36.971 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:36.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:37.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:37.757114   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:37.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:37.757377   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:38.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:38.761199   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:38.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:38.761873   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:38.885 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:38.887 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:38.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:38.888 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:38.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:38.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:38.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:38.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:38.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:39.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:39.765406   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:39.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:39.765715   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:40.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:40.763914   758 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111), 
08:17:40.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:40.764271   758 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:40.769196   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:40.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:40.769443   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:41.626 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:41.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:41.629 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:41.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:41.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:41.631 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:41.631 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:41.631 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:41.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:41.772935   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:41.773177   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:42.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:42.248 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:42.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:42.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:42.248 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:42.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:42.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:42.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:42.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:42.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:42.467 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:42.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:42.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:42.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:42.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:42.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:42.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:42.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:42.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:42.776407   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:42.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:42.776703   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:43.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:43.779613   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:43.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:43.779868   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:43.978801  1029 raft_consensus.cc:482] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:43.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:43.979300  1029 raft_consensus.cc:504] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:43.981207  1029 leader_election.cc:288] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389), 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525)
08:17:43.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:43.983381   779 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:17:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:43.986271   779 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389): Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:43.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:43.986697   781 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525): Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:43.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:43.987074   781 leader_election.cc:302] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 517bd80235974c17827d8c97cb89bece; no voters: 1ded8178b9de45ff8f6d98015be4e05d, 4a70fabb270b4dbcbb09b9e1cac7475d
08:17:43.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:43.987743  1029 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:44.186 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:44.187 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:44.187 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:44.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:44.188 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:44.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:44.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:44.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:44.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:44.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:44.784034   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:44.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:44.784879   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:45.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:45 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069865, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:45.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:45.381942   774 init.cc:295] Successfully reacquired a new kerberos TGT
08:17:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:45.767771   758 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111), 
08:17:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:45.768213   758 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:45.787807   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:45.788125   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:46.790814   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:46.791174   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:47.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:47.087 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:47.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:47.126 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:47.127 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:47.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.128 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.128 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.132 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:47.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:47.552228  1040 raft_consensus.cc:482] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:17:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:47.552997  1040 raft_consensus.cc:504] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1ded8178b9de45ff8f6d98015be4e05d" member_type: VOTER last_known_addr { host: "127.0.118.126" port: 35389 } } peers { permanent_uuid: "4a70fabb270b4dbcbb09b9e1cac7475d" member_type: VOTER last_known_addr { host: "127.0.118.125" port: 46525 } } peers { permanent_uuid: "517bd80235974c17827d8c97cb89bece" member_type: VOTER last_known_addr { host: "127.0.118.124" port: 35773 } }
08:17:47.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:47.555982  1040 leader_election.cc:288] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389), 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525)
08:17:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:47.561825   779 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1ded8178b9de45ff8f6d98015be4e05d (127.0.118.126:35389): Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111)
08:17:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:47.562410   781 leader_election.cc:334] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4a70fabb270b4dbcbb09b9e1cac7475d (127.0.118.125:46525): Network error: Client connection negotiation failed: client connection to 127.0.118.125:46525: connect: Connection refused (error 111)
08:17:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:47.562706   781 leader_election.cc:302] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 517bd80235974c17827d8c97cb89bece; no voters: 1ded8178b9de45ff8f6d98015be4e05d, 4a70fabb270b4dbcbb09b9e1cac7475d
08:17:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:47.563323  1040 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:17:47.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:47.794454   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:47.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:47.794790   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:47.826 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:47.827 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:47.827 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.828 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:47.828 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.828 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:47.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:48.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:48.797583   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:48.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:48.797850   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:49.801028   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:49.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:49.801281   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:50.566 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:50.567 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:50.567 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:50.568 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:50.568 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:50.568 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:50.568 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:50.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:50.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:50.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:50.771018   758 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111), 
08:17:50.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:50.771422   758 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:50.804204   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:50.804474   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:51.807135   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:51.807421   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:52.806 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:52.806 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:52.807 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:52.807 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:52.807 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:52.807 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:52.807 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:52.808 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:52.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:52.811617   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:52.812690   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:53.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:53.815593   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:53.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:53.815919   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:54.819986   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:54.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:54.820314   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:55.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) May 07 08:17:55 dist-test-slave-gbgz krb5kdc[485](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1715069875, etypes {rep=17 tkt=17 ses=17}, kudu/127.0.118.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:17:55.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:55.384646   774 init.cc:295] Successfully reacquired a new kerberos TGT
08:17:55.465 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:55.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:55.466 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:55.467 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:55.467 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:55.467 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:55.467 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:55.467 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:55.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:55.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:55.774870   758 master_runner.cc:186] Error getting master registration for 127.0.118.126:35389: Network error: Client connection negotiation failed: client connection to 127.0.118.126:35389: connect: Connection refused (error 111), 
08:17:55.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:55.775321   758 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:17:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:55.823082   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:55.823462   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:56.825765   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:56.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:56.826022   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:57.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:57.828536   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:57.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:57.828795   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:57.866 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:57.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:57.867 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:57.867 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:57.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:57.868 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:57.868 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:57.868 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:57.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:58.831323   845 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 517bd80235974c17827d8c97cb89bece: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:17:58.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20240507 08:17:58.831569   845 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:17:59.365 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.125:46525(127.0.118.125:46525)] connecting to peer
08:17:59.366 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.0.118.126:35389(127.0.118.126:35389)] connecting to peer
08:17:59.367 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:59.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.0.118.125:46525(127.0.118.125:46525)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:59.367 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.125:46525: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525
08:17:59.367 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:59.367 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.126:35389(127.0.118.126:35389)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:59.367 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.0.118.126:35389: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
08:17:59.369 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389; will retry
08:17:59.374 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29005), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [188ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [245ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [510ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [522ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [524ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [527ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [545ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [548ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [549ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [580ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [581ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [583ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [600ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [602ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [605ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [624ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [625ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [680ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [681ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [683ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [721ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [723ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [724ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [725ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [880ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [881ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [883ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1076ms] refreshing cache from master, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1084ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1316ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1322ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1436ms] refreshing cache from master, [1436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1442ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1442ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [4316ms] refreshing cache from master, [4316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [4317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [4318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [4319ms] too many traces: truncated at 100 traces, deferred=Deferred@474260656(state=PENDING, result=null, callback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251), errback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251)))
08:17:59.378 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.0.118.124:35773(127.0.118.124:35773)] disconnecting while in state READY
08:17:59.379 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.0.118.126:35389(127.0.118.126:35389)] disconnecting while in state TERMINATED
08:17:59.379 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.0.118.125:46525(127.0.118.125:46525)] disconnecting while in state TERMINATED
08:17:59.384 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:17:59.386 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.0.118.124:35773(127.0.118.124:35773)] handling channelInactive
08:17:59.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.0.118.124:35773(127.0.118.124:35773)] cleaning up while in state READY due to: connection closed
08:17:59.388 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:17:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:17:59.392649   473 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 758
08:18:00.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-07T08:18:00Z chronyd exiting
08:18:01.525 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:18:01.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:18:01.526546 32220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 32518
08:18:01.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:18:01.553339 32220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 32656
08:18:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:18:01.582530 32220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 331
08:18:01.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:18:01.612221 32220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 32232
08:18:01.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:18:01.643925 32220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 32302
08:18:01.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20240507 08:18:01.676638 32220 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskN8qzGC/build/tsan/bin/kudu with pid 32377
08:18:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2024-05-07T08:18:02Z chronyd exiting
08:18:03.787 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29005), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [188ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [245ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [510ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [522ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [524ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [527ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [545ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [548ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [549ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [580ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [581ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [583ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [600ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [602ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [605ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [624ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [625ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [680ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [681ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [683ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [721ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [723ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [724ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [725ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [880ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [881ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [883ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1076ms] refreshing cache from master, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1084ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1316ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1322ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1436ms] refreshing cache from master, [1436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1442ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1442ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [4316ms] refreshing cache from master, [4316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [4317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [4318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [4319ms] too many traces: truncated at 100 traces, deferred=Deferred@474260656(state=PENDING, result=null, callback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251), errback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.118.126:35389
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29005), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [188ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [245ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [510ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [522ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [524ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [527ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [545ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [548ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [549ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [580ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [581ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [583ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [600ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [602ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [605ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [624ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [625ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [680ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [681ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [683ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [721ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [723ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [724ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [725ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [880ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [881ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [883ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1076ms] refreshing cache from master, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1084ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1316ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1322ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1436ms] refreshing cache from master, [1436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1442ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1442ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [4316ms] refreshing cache from master, [4316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [4317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [4318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [4319ms] too many traces: truncated at 100 traces, deferred=Deferred@474260656(state=PENDING, result=null, callback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251), errback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.118.126:35389
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

08:18:04.274 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
08:18:04.275 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29005), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [188ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [245ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [510ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [522ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [524ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [527ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [545ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [548ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [549ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [580ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [581ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [583ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [600ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [602ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [605ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [624ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [625ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [680ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [681ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [683ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [721ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [723ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [724ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [725ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [880ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [881ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [883ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1076ms] refreshing cache from master, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1084ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1316ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1322ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1436ms] refreshing cache from master, [1436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1442ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1442ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [4316ms] refreshing cache from master, [4316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [4317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [4318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [4319ms] too many traces: truncated at 100 traces, deferred=Deferred@474260656(state=PENDING, result=null, callback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251), errback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.118.126:35389
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29005), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [188ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [245ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [510ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [514ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [516ms] refreshing cache from master, [516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [518ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [520ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [522ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [524ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [526ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [527ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [536ms] refreshing cache from master, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [545ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [546ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [548ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [549ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [575ms] refreshing cache from master, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [576ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [580ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [581ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [582ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [583ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [596ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [600ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [602ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [604ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [605ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [616ms] refreshing cache from master, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [624ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [625ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [680ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [681ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [682ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [683ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [721ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [723ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [724ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [725ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [876ms] refreshing cache from master, [876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [880ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [881ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [882ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [883ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1076ms] refreshing cache from master, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1084ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1316ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1319ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1322ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1436ms] refreshing cache from master, [1436ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [1437ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.125:46525: Network error: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525, [1440ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.124:35773: OK, [1442ms] Sub RPC ConnectToMaster: received response from server master-127.0.118.126:35389: Network error: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [1442ms] delaying RPC due to: Service unavailable: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389, [4316ms] refreshing cache from master, [4316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.124:35773, [4317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.125:46525, [4318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.0.118.126:35389, [4319ms] too many traces: truncated at 100 traces, deferred=Deferred@474260656(state=PENDING, result=null, callback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251), errback=(continuation of Deferred@958363348 after retry RPC after error@520840521) -> (continuation of Deferred@1963454662 after retry RPC after error@1281338354) -> (continuation of Deferred@246730107 after retry RPC after error@1255264457) -> (continuation of Deferred@10127509 after retry RPC after error@833336143) -> (continuation of Deferred@425492244 after retry RPC after error@1565504366) -> (continuation of Deferred@155534962 after retry RPC after error@1696116190) -> (continuation of Deferred@524733110 after retry RPC after error@1068687091) -> (continuation of Deferred@476543393 after retry RPC after error@1009861038) -> (continuation of Deferred@353058788 after retry RPC after error@1021783792) -> (continuation of Deferred@468382396 after retry RPC after error@769715294) -> (continuation of Deferred@1641550361 after retry RPC after error@1682776188) -> (continuation of Deferred@687751515 after retry RPC after error@47630854) -> (continuation of Deferred@591354015 after retry RPC after error@779928132) -> (continuation of Deferred@1959967305 after retry RPC after error@1088477231) -> (continuation of Deferred@423791372 after retry RPC after error@1365509250) -> (continuation of Deferred@805698581 after retry RPC after error@1986165658) -> (continuation of Deferred@2071299249 after retry RPC after error@172285769) -> (continuation of Deferred@364801856 after retry RPC after error@1885698246) -> (continuation of Deferred@669133431 after retry RPC after error@191895381) -> (continuation of Deferred@1834134804 after retry RPC after error@917130032) -> (continuation of Deferred@11365888 after retry RPC after error@191630335) -> (continuation of Deferred@1940794763 after retry RPC after error@389618437) -> (continuation of Deferred@2081401484 after retry RPC after error@399996161) -> (continuation of Deferred@353878021 after retry RPC after error@1244205201) -> (continuation of Deferred@1274255506 after retry RPC after error@1730188176) -> (continuation of Deferred@1285320857 after retry RPC after error@1320373251)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.0.118.124:35773,127.0.118.125:46525,127.0.118.126:35389) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.125:46525(127.0.118.125:46525): Connection refused: /127.0.118.125:46525,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.0.118.126:35389(127.0.118.126:35389): Connection refused: /127.0.118.126:35389
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.0.118.126:35389
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)