Diagnosed failure

Unable to diagnose
Full log

08:10:17.105 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:10:17.105 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4Hf2Nr/test-tmp
08:10:17.106 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:10:17.106 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4Hf2Nr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:10:17.107 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

08:10:17.423 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:10:17.424 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

08:10:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.426819 13731 mini_oidc.cc:150] Starting JWKS server
08:10:17.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.430130 13731 webserver.cc:480] Webserver started at http://127.0.0.1:41079/ using document root <none> and password file <none>
08:10:17.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.430537 13731 mini_oidc.cc:186] Starting OIDC Discovery server
08:10:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.432122 13731 webserver.cc:480] Webserver started at http://127.0.0.1:42207/ using document root <none> and password file <none>
08:10:17.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:10:17Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:10:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:10:17Z Disabled control of system clock
08:10:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.492507 13731 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:17.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/wal
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/logs
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data/info.pb
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.254:36695
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.254
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41079/jwks.json with env {}
08:10:17.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.781594 13743 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:17.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.782145 13743 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:17.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.782792 13743 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.815161 13743 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.815459 13743 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.815670 13743 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.815876 13743 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.816062 13743 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:17.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.816215 13743 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41079/jwks.json
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.851567 13743 master_runner.cc:386] Master server non-default flags:
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/wal
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.254:36695
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41079/jwks.json
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data/info.pb
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.254
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/logs
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:17.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:17.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.852833 13743 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:17.854370 13743 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:17.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.864792 13749 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:17.865680 13750 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.268898 13748 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 13743
08:10:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.637949 13743 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.773s	user 0.600s	sys 1.172s
08:10:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.639113 13743 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.774s	user 0.601s	sys 1.173s
08:10:19.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.640682 13752 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.643152 13751 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1773 milliseconds
08:10:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.643167 13743 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.644420 13743 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:19.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.646775 13743 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:19.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.648159 13743 hybrid_clock.cc:648] HybridClock initialized: now 1756973419648119 us; error 50 us; skew 500 ppm
08:10:19.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.648908 13743 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.654765 13743 webserver.cc:480] Webserver started at http://127.13.104.254:43101/ using document root <none> and password file <none>
08:10:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.655661 13743 fs_manager.cc:362] Metadata directory not provided
08:10:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.655843 13743 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.656221 13743 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.660665 13743 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data/instance:
08:10:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad26cc27943b4ece9bc39f85ff4c51cf"
08:10:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:19 on dist-test-slave-q46p"
08:10:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.661648 13743 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/wal/instance:
08:10:19.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad26cc27943b4ece9bc39f85ff4c51cf"
08:10:19.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:19 on dist-test-slave-q46p"
08:10:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.668566 13743 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.004s	sys 0.004s
08:10:19.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.673631 13759 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.674602 13743 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.002s	sys 0.001s
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.674906 13743 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/wal
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ad26cc27943b4ece9bc39f85ff4c51cf"
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:19 on dist-test-slave-q46p"
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.675190 13743 fs_report.cc:389] FS layout report
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/wal
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/wal
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data/data
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.722587 13743 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.724135 13743 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:19.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.724550 13743 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:19.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.796475 13743 rpc_server.cc:307] RPC server started. Bound to: 127.13.104.254:36695
08:10:19.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.796525 13810 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.104.254:36695 every 8 connection(s)
08:10:19.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.799083 13743 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/data/info.pb
08:10:19.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.802626 13731 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 13743
08:10:19.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.803175 13731 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-0/wal/instance
08:10:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.804999 13811 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:10:19.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.808449 13731 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/wal
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/logs
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data/info.pb
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.253:45325
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.253
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41079/jwks.json with env {}
08:10:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.819283 13811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } has no permanent_uuid. Determining permanent_uuid...
08:10:19.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.844771 13811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } has no permanent_uuid. Determining permanent_uuid...
08:10:19.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.847107 13762 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:19.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.851327 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.253:45325: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.900884 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } attempt: 1
08:10:19.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.904773 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.253:45325: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:19.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:19.973289 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } attempt: 2
08:10:19.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:19.977198 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.253:45325: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:20.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:20.068686 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } attempt: 3
08:10:20.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.072597 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.253:45325: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.111918 13813 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:20.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.112457 13813 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.112908 13813 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.143227 13813 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.143564 13813 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.143770 13813 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.143962 13813 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.144150 13813 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:20.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.144292 13813 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41079/jwks.json
08:10:20.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:20.178630 13813 master_runner.cc:386] Master server non-default flags:
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/wal
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.253:45325
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41079/jwks.json
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data/info.pb
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.253
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/logs
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:20.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:20.179929 13813 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:20.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:20.181437 13813 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:20.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.191772 13822 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:20.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:20.216069 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } attempt: 4
08:10:20.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.222713 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.253:45325: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:20.522336 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } attempt: 5
08:10:20.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.528914 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.253:45325: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:21.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.076464 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } attempt: 6
08:10:21.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.086793 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.253:45325: Network error: Client connection negotiation failed: client connection to 127.13.104.253:45325: connect: Connection refused (error 111)
08:10:21.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:20.192756 13823 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:21.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.378921 13825 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.381232 13824 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1184 milliseconds
08:10:21.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.381379 13813 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:21.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.382608 13813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:21.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.385533 13813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.386965 13813 hybrid_clock.cc:648] HybridClock initialized: now 1756973421386915 us; error 51 us; skew 500 ppm
08:10:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.387765 13813 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:21.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.394112 13813 webserver.cc:480] Webserver started at http://127.13.104.253:45015/ using document root <none> and password file <none>
08:10:21.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.394999 13813 fs_manager.cc:362] Metadata directory not provided
08:10:21.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.395191 13813 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:21.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.395573 13813 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.399860 13813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data/instance:
08:10:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "911be842d56c44ba9c697a3eb53a7f33"
08:10:21.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:21 on dist-test-slave-q46p"
08:10:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.400840 13813 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/wal/instance:
08:10:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "911be842d56c44ba9c697a3eb53a7f33"
08:10:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:21 on dist-test-slave-q46p"
08:10:21.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.407703 13813 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.001s
08:10:21.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.413148 13834 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.414192 13813 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.005s	sys 0.000s
08:10:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.414537 13813 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/wal
08:10:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "911be842d56c44ba9c697a3eb53a7f33"
08:10:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:21 on dist-test-slave-q46p"
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.414894 13813 fs_report.cc:389] FS layout report
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/wal
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/wal
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data/data
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:21.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.469533 13813 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:21.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.471079 13813 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:21.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.471563 13813 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.539942 13813 rpc_server.cc:307] RPC server started. Bound to: 127.13.104.253:45325
08:10:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.540019 13885 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.104.253:45325 every 8 connection(s)
08:10:21.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.542533 13813 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/data/info.pb
08:10:21.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.543275 13731 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 13813
08:10:21.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.543638 13731 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-1/wal/instance
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.546614 13731 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/wal
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/logs
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data/info.pb
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.252:44239
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.252
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41079/jwks.json with env {}
08:10:21.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.548955 13886 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:10:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.560465 13886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } has no permanent_uuid. Determining permanent_uuid...
08:10:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.585487 13886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } has no permanent_uuid. Determining permanent_uuid...
08:10:21.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.598713 13886 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } has no permanent_uuid. Determining permanent_uuid...
08:10:21.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.601227 13838 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.605265 13886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:21.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.654780 13886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 1
08:10:21.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.658557 13886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:21.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.726922 13886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 2
08:10:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.730820 13886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.822247 13886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 3
08:10:21.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.826074 13886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.861874 13888 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:21.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.862560 13888 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.863180 13888 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.893815 13888 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.894163 13888 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.894488 13888 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.894757 13888 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.895063 13888 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:21.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.895284 13888 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41079/jwks.json
08:10:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.930518 13888 master_runner.cc:386] Master server non-default flags:
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/wal
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.252:44239
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41079/jwks.json
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data/info.pb
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.252
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/logs
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.932184 13888 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:21.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.933868 13888 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:21.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.944319 13898 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:21.969447 13886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 4
08:10:21.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.977385 13886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.147464 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } attempt: 7
08:10:22.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.197921 13811 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } has no permanent_uuid. Determining permanent_uuid...
08:10:22.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.219151 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.277007 13886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 5
08:10:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.281306 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 1
08:10:22.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.287057 13886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.296110 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.377719 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 2
08:10:22.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.386085 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.471709 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 3
08:10:22.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.480271 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.620896 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 4
08:10:22.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.629410 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.834645 13886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 6
08:10:22.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.857113 13886 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:22.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:22.913009 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 5
08:10:22.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:22.921005 13811 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.104.252:44239: Network error: Client connection negotiation failed: client connection to 127.13.104.252:44239: connect: Connection refused (error 111)
08:10:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:21.945082 13899 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.186460 13901 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:23.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.190547 13888 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.246s	user 0.429s	sys 0.797s
08:10:23.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.191083 13888 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.246s	user 0.431s	sys 0.799s
08:10:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.193825 13900 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1240 milliseconds
08:10:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.193876 13888 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.195158 13888 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:23.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.197690 13888 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.199030 13888 hybrid_clock.cc:648] HybridClock initialized: now 1756973423199009 us; error 59 us; skew 500 ppm
08:10:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.199896 13888 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.207170 13888 webserver.cc:480] Webserver started at http://127.13.104.252:39725/ using document root <none> and password file <none>
08:10:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.208108 13888 fs_manager.cc:362] Metadata directory not provided
08:10:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.208369 13888 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:23.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.208870 13888 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.213410 13888 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data/instance:
08:10:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6686c793b90481ca5fa67d69d6ab586"
08:10:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:23 on dist-test-slave-q46p"
08:10:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.214504 13888 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/wal/instance:
08:10:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6686c793b90481ca5fa67d69d6ab586"
08:10:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:23 on dist-test-slave-q46p"
08:10:23.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.222317 13888 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.005s	sys 0.001s
08:10:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.228346 13913 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:23.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.229475 13888 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.000s	sys 0.005s
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.229772 13888 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/wal
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6686c793b90481ca5fa67d69d6ab586"
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:23 on dist-test-slave-q46p"
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.230057 13888 fs_report.cc:389] FS layout report
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/wal
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/wal
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data/data
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:23.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:23.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.303898 13888 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.305363 13888 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:23.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.305769 13888 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.375463 13888 rpc_server.cc:307] RPC server started. Bound to: 127.13.104.252:44239
08:10:23.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.375519 13964 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.104.252:44239 every 8 connection(s)
08:10:23.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.378221 13888 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/data/info.pb
08:10:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.381965 13731 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 13888
08:10:23.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.382481 13731 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/master-2/wal/instance
08:10:23.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.384301 13965 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:10:23.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.400825 13965 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } has no permanent_uuid. Determining permanent_uuid...
08:10:23.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.427440 13965 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } has no permanent_uuid. Determining permanent_uuid...
08:10:23.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.437458 13965 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } has no permanent_uuid. Determining permanent_uuid...
08:10:23.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.453510 13965 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586: Bootstrap starting.
08:10:23.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.459312 13965 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586: Neither blocks nor log segments found. Creating new log.
08:10:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.461340 13965 log.cc:826] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586: Log is configured to *not* fsync() on all Append() calls
08:10:23.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.465991 13965 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586: No bootstrap required, opened a new log
08:10:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.473567 13811 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 6
08:10:23.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.487851 13965 raft_consensus.cc:357] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:23.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.488461 13965 raft_consensus.cc:383] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:10:23.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.488682 13965 raft_consensus.cc:738] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d6686c793b90481ca5fa67d69d6ab586, State: Initialized, Role: FOLLOWER
08:10:23.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.489282 13965 consensus_queue.cc:260] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.492017 13811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: Bootstrap starting.
08:10:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.491822 13975 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.492592 13975 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: This master's current role is: FOLLOWER
08:10:23.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.493436 13965 sys_catalog.cc:564] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: configured and running, proceeding with master startup.
08:10:23.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.500178 13811 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: Neither blocks nor log segments found. Creating new log.
08:10:23.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.502704 13811 log.cc:826] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: Log is configured to *not* fsync() on all Append() calls
08:10:23.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.507761 13811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: No bootstrap required, opened a new log
08:10:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.524456 13987 catalog_manager.cc:1560] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.524825 13987 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.534720 13811 raft_consensus.cc:357] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:23.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.535645 13811 raft_consensus.cc:383] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:10:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.535928 13811 raft_consensus.cc:738] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ad26cc27943b4ece9bc39f85ff4c51cf, State: Initialized, Role: FOLLOWER
08:10:23.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.536936 13811 consensus_queue.cc:260] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.519114 13888 master_runner.cc:186] Error getting master registration for 127.13.104.254:36695: OK, instance_id { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" instance_seqno: 1756973419776563 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:10:23.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.557894 13888 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:10:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.559495 13989 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:23.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.560457 13989 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [sys.catalog]: This master's current role is: FOLLOWER
08:10:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.560784 13919 tablet.cc:2378] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:10:23.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.561632 13811 sys_catalog.cc:564] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [sys.catalog]: configured and running, proceeding with master startup.
08:10:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.602492 14000 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.603171 14000 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.611109 13975 raft_consensus.cc:491] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:10:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.611629 13975 raft_consensus.cc:513] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.614104 13975 leader_election.cc:290] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ad26cc27943b4ece9bc39f85ff4c51cf (127.13.104.254:36695), 911be842d56c44ba9c697a3eb53a7f33 (127.13.104.253:45325)
08:10:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.616642 13786 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6686c793b90481ca5fa67d69d6ab586" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" is_pre_election: true
08:10:23.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.617630 13786 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d6686c793b90481ca5fa67d69d6ab586 in term 0.
08:10:23.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.619058 13916 leader_election.cc:304] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad26cc27943b4ece9bc39f85ff4c51cf, d6686c793b90481ca5fa67d69d6ab586; no voters: 
08:10:23.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.620007 13975 raft_consensus.cc:2802] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:10:23.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.620326 13975 raft_consensus.cc:491] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:10:23.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.620688 13975 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 0 FOLLOWER]: Advancing to term 1
08:10:23.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.627816 13975 raft_consensus.cc:513] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:23.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.630544 13786 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6686c793b90481ca5fa67d69d6ab586" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf"
08:10:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.631165 13786 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [term 0 FOLLOWER]: Advancing to term 1
08:10:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.600236 13743 master_runner.cc:186] Error getting master registration for 127.13.104.253:45325: OK, instance_id { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" instance_seqno: 1756973421521800 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:10:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.634531 13743 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:10:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.636185 13975 leader_election.cc:290] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [CANDIDATE]: Term 1 election: Requested vote from peers ad26cc27943b4ece9bc39f85ff4c51cf (127.13.104.254:36695), 911be842d56c44ba9c697a3eb53a7f33 (127.13.104.253:45325)
08:10:23.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.638476 13786 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d6686c793b90481ca5fa67d69d6ab586 in term 1.
08:10:23.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.639672 13916 leader_election.cc:304] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ad26cc27943b4ece9bc39f85ff4c51cf, d6686c793b90481ca5fa67d69d6ab586; no voters: 
08:10:23.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.644064 13975 raft_consensus.cc:2802] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 1 FOLLOWER]: Leader election won for term 1
08:10:23.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.646486 13975 raft_consensus.cc:695] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [term 1 LEADER]: Becoming Leader. State: Replica: d6686c793b90481ca5fa67d69d6ab586, State: Running, Role: LEADER
08:10:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.647477 13975 consensus_queue.cc:237] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:23.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.656394 14002 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d6686c793b90481ca5fa67d69d6ab586. Latest consensus state: current_term: 1 leader_uuid: "d6686c793b90481ca5fa67d69d6ab586" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:23.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.657222 14002 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: This master's current role is: LEADER
08:10:23.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.666182 14004 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:10:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.675247 14004 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:10:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.702678 13786 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [term 1 FOLLOWER]: Refusing update from remote peer d6686c793b90481ca5fa67d69d6ab586: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:10:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.705793 13975 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:10:23.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:23.733880 13765 tablet.cc:2378] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:10:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.736320 13989 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [sys.catalog]: SysCatalogTable state changed. Reason: New leader d6686c793b90481ca5fa67d69d6ab586. Latest consensus state: current_term: 1 leader_uuid: "d6686c793b90481ca5fa67d69d6ab586" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:23.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.739060 13989 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [sys.catalog]: This master's current role is: FOLLOWER
08:10:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.745323 14005 mvcc.cc:204] Tried to move back new op lower bound from 7196563143458004992 to 7196563143270203392. Current Snapshot: MvccSnapshot[applied={T|T < 7196563143458004992}]
08:10:23.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.748961 14006 mvcc.cc:204] Tried to move back new op lower bound from 7196563143458004992 to 7196563143270203392. Current Snapshot: MvccSnapshot[applied={T|T < 7196563143458004992}]
08:10:23.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.756428 13989 sys_catalog.cc:455] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d6686c793b90481ca5fa67d69d6ab586" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:23.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.757925 13989 sys_catalog.cc:458] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf [sys.catalog]: This master's current role is: FOLLOWER
08:10:23.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.767104 14004 catalog_manager.cc:1349] Generated new cluster ID: 000b99e0c0ee466f890d8892b030d307
08:10:23.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.767426 14004 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:10:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.771179 13975 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d6686c793b90481ca5fa67d69d6ab586" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.771881 13975 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: This master's current role is: LEADER
08:10:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.812280 14004 catalog_manager.cc:1372] Generated new certificate authority record
08:10:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.815722 14004 catalog_manager.cc:1506] Loading token signing keys...
08:10:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.842255 14004 catalog_manager.cc:5955] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586: Generated new TSK 0
08:10:23.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.843478 14004 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.864180 13731 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/wal
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/logs
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data/info.pb
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.193:0
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.104.193
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.193
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:23.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:10:23.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.917668 13886 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } attempt: 7
08:10:23.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.953950 13886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33: Bootstrap starting.
08:10:23.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.961432 13886 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33: Neither blocks nor log segments found. Creating new log.
08:10:23.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.964051 13886 log.cc:826] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33: Log is configured to *not* fsync() on all Append() calls
08:10:23.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:23.993279 13886 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33: No bootstrap required, opened a new log
08:10:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.068394 13886 raft_consensus.cc:357] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.071753 13886 raft_consensus.cc:383] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:10:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.072708 13886 raft_consensus.cc:738] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 911be842d56c44ba9c697a3eb53a7f33, State: Initialized, Role: FOLLOWER
08:10:24.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.075337 13886 consensus_queue.cc:260] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } }
08:10:24.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.098544 13860 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6686c793b90481ca5fa67d69d6ab586" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "911be842d56c44ba9c697a3eb53a7f33"
08:10:24.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.098727 13861 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d6686c793b90481ca5fa67d69d6ab586" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "911be842d56c44ba9c697a3eb53a7f33" is_pre_election: true
08:10:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.099303 13859 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [term 0 FOLLOWER]: Advancing to term 1
08:10:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.112385 13859 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [term 1 FOLLOWER]: Refusing update from remote peer d6686c793b90481ca5fa67d69d6ab586: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:10:24.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.114135 13975 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [LEADER]: Connected to new peer: Peer: permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:10:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.098208 14013 sys_catalog.cc:455] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.123561 14013 sys_catalog.cc:458] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [sys.catalog]: This master's current role is: FOLLOWER
08:10:24.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.121680 13886 sys_catalog.cc:564] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [sys.catalog]: configured and running, proceeding with master startup.
08:10:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.193442 14013 sys_catalog.cc:455] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d6686c793b90481ca5fa67d69d6ab586. Latest consensus state: current_term: 1 leader_uuid: "d6686c793b90481ca5fa67d69d6ab586" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.194931 14014 sys_catalog.cc:455] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d6686c793b90481ca5fa67d69d6ab586" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.195878 14014 sys_catalog.cc:458] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [sys.catalog]: This master's current role is: FOLLOWER
08:10:24.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.196533 14013 sys_catalog.cc:458] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33 [sys.catalog]: This master's current role is: FOLLOWER
08:10:24.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.231352 14029 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:10:24.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:24.416318 14010 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:24.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:24.417290 14010 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:24.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:24.418218 14010 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:24.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:24.479805 14010 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:24.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:24.481222 14010 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.104.193
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.546509 14010 tablet_server_runner.cc:78] Tablet server non-default flags:
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/wal
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.193:0
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data/info.pb
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.193
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.104.193
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/logs
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:24.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:24.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.548676 14010 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:24.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.550812 14010 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:24.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.556632 13975 sys_catalog.cc:455] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d6686c793b90481ca5fa67d69d6ab586" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ad26cc27943b4ece9bc39f85ff4c51cf" member_type: VOTER last_known_addr { host: "127.13.104.254" port: 36695 } } peers { permanent_uuid: "911be842d56c44ba9c697a3eb53a7f33" member_type: VOTER last_known_addr { host: "127.13.104.253" port: 45325 } } peers { permanent_uuid: "d6686c793b90481ca5fa67d69d6ab586" member_type: VOTER last_known_addr { host: "127.13.104.252" port: 44239 } } }
08:10:24.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.557478 13975 sys_catalog.cc:458] T 00000000000000000000000000000000 P d6686c793b90481ca5fa67d69d6ab586 [sys.catalog]: This master's current role is: LEADER
08:10:24.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:24.587114 14037 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.609180 14000 catalog_manager.cc:1261] Loaded cluster ID: 000b99e0c0ee466f890d8892b030d307
08:10:24.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.609704 14000 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: loading cluster ID for follower catalog manager: success
08:10:24.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.617448 14000 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: acquiring CA information for follower catalog manager: success
08:10:24.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:24.637092 14000 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ad26cc27943b4ece9bc39f85ff4c51cf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:10:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:25.247886 14029 catalog_manager.cc:1261] Loaded cluster ID: 000b99e0c0ee466f890d8892b030d307
08:10:25.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:25.248565 14029 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33: loading cluster ID for follower catalog manager: success
08:10:25.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:25.260938 14029 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33: acquiring CA information for follower catalog manager: success
08:10:25.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:25.270061 14029 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 911be842d56c44ba9c697a3eb53a7f33: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:10:26.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:25.982906 14036 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 14010
08:10:26.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:26.311487 14036 kernel_stack_watchdog.cc:198] Thread 14010 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 390ms:
08:10:26.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:10:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:10:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:10:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:10:26.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:24.592092 14038 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:26.314574 14010 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.724s	user 0.552s	sys 0.993s
08:10:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:26.315105 14010 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.725s	user 0.553s	sys 0.993s
08:10:26.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.321419 14010 server_base.cc:1047] running on GCE node
08:10:26.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:26.322768 14043 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:26.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.324263 14010 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:26.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.335345 14010 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:26.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.337409 14010 hybrid_clock.cc:648] HybridClock initialized: now 1756973426337366 us; error 57 us; skew 500 ppm
08:10:26.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.338497 14010 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:26.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.383798 14010 webserver.cc:480] Webserver started at http://127.13.104.193:46099/ using document root <none> and password file <none>
08:10:26.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.390576 14010 fs_manager.cc:362] Metadata directory not provided
08:10:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.390950 14010 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:26.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.391597 14010 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.399022 14010 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data/instance:
08:10:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77c19d3b757144b28a4872e77fe83dc1"
08:10:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:26 on dist-test-slave-q46p"
08:10:26.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.400524 14010 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/wal/instance:
08:10:26.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77c19d3b757144b28a4872e77fe83dc1"
08:10:26.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:26 on dist-test-slave-q46p"
08:10:26.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.429158 14010 fs_manager.cc:696] Time spent creating directory manager: real 0.028s	user 0.002s	sys 0.024s
08:10:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.447866 14048 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:26.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.453406 14010 fs_manager.cc:730] Time spent opening block manager: real 0.019s	user 0.008s	sys 0.008s
08:10:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.454020 14010 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/wal
08:10:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77c19d3b757144b28a4872e77fe83dc1"
08:10:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:26 on dist-test-slave-q46p"
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.454627 14010 fs_report.cc:389] FS layout report
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/wal
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/wal
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data/data
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:26.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.560310 14010 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:26.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.562391 14010 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.563092 14010 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:26.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.586580 14010 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:10:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.599965 14010 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:10:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.600942 14010 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s	user 0.000s	sys 0.000s
08:10:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.601393 14010 ts_tablet_manager.cc:610] Registered 0 tablets
08:10:26.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:26.601715 14010 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:10:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.199383 14010 rpc_server.cc:307] RPC server started. Bound to: 127.13.104.193:42603
08:10:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.199522 14160 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.104.193:42603 every 8 connection(s)
08:10:27.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.203292 14010 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/data/info.pb
08:10:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.205204 13731 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14010
08:10:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.205734 13731 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-0/wal/instance
08:10:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.224188 13731 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/wal
08:10:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data
08:10:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:27.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/logs
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data/info.pb
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.194:0
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.104.194
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.194
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:27.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:10:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.322203 14162 heartbeater.cc:344] Connected to a master server at 127.13.104.252:44239
08:10:27.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.322742 14162 heartbeater.cc:461] Registering TS with master...
08:10:27.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.324083 14162 heartbeater.cc:507] Master 127.13.104.252:44239 requested a full tablet report, sending...
08:10:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.327746 13930 ts_manager.cc:194] Registered new tserver with Master: 77c19d3b757144b28a4872e77fe83dc1 (127.13.104.193:42603)
08:10:27.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.330296 13930 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.104.193:48705
08:10:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.332962 14161 heartbeater.cc:344] Connected to a master server at 127.13.104.253:45325
08:10:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.333333 14161 heartbeater.cc:461] Registering TS with master...
08:10:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.334249 14161 heartbeater.cc:507] Master 127.13.104.253:45325 requested a full tablet report, sending...
08:10:27.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.337245 13851 ts_manager.cc:194] Registered new tserver with Master: 77c19d3b757144b28a4872e77fe83dc1 (127.13.104.193:42603)
08:10:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.347612 14166 heartbeater.cc:344] Connected to a master server at 127.13.104.254:36695
08:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.347955 14166 heartbeater.cc:461] Registering TS with master...
08:10:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.348570 14166 heartbeater.cc:507] Master 127.13.104.254:36695 requested a full tablet report, sending...
08:10:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.351374 13776 ts_manager.cc:194] Registered new tserver with Master: 77c19d3b757144b28a4872e77fe83dc1 (127.13.104.193:42603)
08:10:27.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.589306 14165 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:27.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.589782 14165 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.590212 14165 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.622627 14165 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.623590 14165 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.104.194
08:10:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.659374 14165 tablet_server_runner.cc:78] Tablet server non-default flags:
08:10:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/wal
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.194:0
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data/info.pb
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.194
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.104.194
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/logs
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:27.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.660694 14165 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:27.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:27.662315 14165 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:27.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.675468 14178 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:28.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:28.344177 14162 heartbeater.cc:499] Master 127.13.104.252:44239 was elected leader, sending a full tablet report...
08:10:29.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.677177 14179 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:29.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:27.681033 14181 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.074745 14180 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1394 milliseconds
08:10:29.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.074903 14165 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:29.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.076155 14165 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:29.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.078830 14165 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.080291 14165 hybrid_clock.cc:648] HybridClock initialized: now 1756973429080248 us; error 62 us; skew 500 ppm
08:10:29.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.081221 14165 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:29.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.087983 14165 webserver.cc:480] Webserver started at http://127.13.104.194:33171/ using document root <none> and password file <none>
08:10:29.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.088898 14165 fs_manager.cc:362] Metadata directory not provided
08:10:29.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.089097 14165 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:29.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.089499 14165 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.094345 14165 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data/instance:
08:10:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33a79d9eb78c4c469b23e0528d678608"
08:10:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:29 on dist-test-slave-q46p"
08:10:29.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.095769 14165 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/wal/instance:
08:10:29.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33a79d9eb78c4c469b23e0528d678608"
08:10:29.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:29 on dist-test-slave-q46p"
08:10:29.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.103106 14165 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.005s
08:10:29.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.108772 14189 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.109841 14165 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.003s
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.110134 14165 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/wal
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33a79d9eb78c4c469b23e0528d678608"
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:29 on dist-test-slave-q46p"
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.110424 14165 fs_report.cc:389] FS layout report
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/wal
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/wal
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data/data
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:29.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:29.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:29.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.163700 14165 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.165318 14165 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.165853 14165 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:29.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.168349 14165 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:10:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.172300 14165 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:10:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.172510 14165 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:10:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.172799 14165 ts_tablet_manager.cc:610] Registered 0 tablets
08:10:29.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.172983 14165 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:10:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.312832 14165 rpc_server.cc:307] RPC server started. Bound to: 127.13.104.194:43919
08:10:29.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.312951 14301 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.104.194:43919 every 8 connection(s)
08:10:29.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.315282 14165 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/data/info.pb
08:10:29.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.316097 13731 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14165
08:10:29.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.316725 13731 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-1/wal/instance
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.332795 13731 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/wal
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/logs
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data/info.pb
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:29.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.195:0
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.104.195
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.195
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:10:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.371335 14303 heartbeater.cc:344] Connected to a master server at 127.13.104.252:44239
08:10:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.371627 14302 heartbeater.cc:344] Connected to a master server at 127.13.104.253:45325
08:10:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.371868 14303 heartbeater.cc:461] Registering TS with master...
08:10:29.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.372200 14302 heartbeater.cc:461] Registering TS with master...
08:10:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.373329 14303 heartbeater.cc:507] Master 127.13.104.252:44239 requested a full tablet report, sending...
08:10:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.374347 14304 heartbeater.cc:344] Connected to a master server at 127.13.104.254:36695
08:10:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.374650 14304 heartbeater.cc:461] Registering TS with master...
08:10:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.375525 14304 heartbeater.cc:507] Master 127.13.104.254:36695 requested a full tablet report, sending...
08:10:29.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.376247 13930 ts_manager.cc:194] Registered new tserver with Master: 33a79d9eb78c4c469b23e0528d678608 (127.13.104.194:43919)
08:10:29.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.377869 13930 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.104.194:55021
08:10:29.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.378167 13776 ts_manager.cc:194] Registered new tserver with Master: 33a79d9eb78c4c469b23e0528d678608 (127.13.104.194:43919)
08:10:29.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.373365 14302 heartbeater.cc:507] Master 127.13.104.253:45325 requested a full tablet report, sending...
08:10:29.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.381305 13851 ts_manager.cc:194] Registered new tserver with Master: 33a79d9eb78c4c469b23e0528d678608 (127.13.104.194:43919)
08:10:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.661967 14310 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.662525 14310 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:29.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.663102 14310 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.694110 14310 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:29.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.695099 14310 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.104.195
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.730612 14310 tablet_server_runner.cc:78] Tablet server non-default flags:
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.104.212:45457
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/wal
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.104.195:0
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data/info.pb
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.104.195
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.104.254:36695,127.13.104.253:45325,127.13.104.252:44239
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.104.195
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/logs
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:29.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.732275 14310 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:29.733995 14310 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:29.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.746440 14318 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:30.382303 14303 heartbeater.cc:499] Master 127.13.104.252:44239 was elected leader, sending a full tablet report...
08:10:30.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.748150 14319 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:30.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:29.750541 14321 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:30.993052 14320 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1240 milliseconds
08:10:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:30.993149 14310 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:30.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:30.994264 14310 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:30.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:30.996907 14310 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:30.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:30.998311 14310 hybrid_clock.cc:648] HybridClock initialized: now 1756973430998271 us; error 55 us; skew 500 ppm
08:10:30.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:30.999121 14310 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.005728 14310 webserver.cc:480] Webserver started at http://127.13.104.195:46073/ using document root <none> and password file <none>
08:10:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.006579 14310 fs_manager.cc:362] Metadata directory not provided
08:10:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.006790 14310 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:31.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.007247 14310 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.011638 14310 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data/instance:
08:10:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67f6c9330d05405d8014fc5778fc8dc6"
08:10:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:31 on dist-test-slave-q46p"
08:10:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.012674 14310 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/wal/instance:
08:10:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67f6c9330d05405d8014fc5778fc8dc6"
08:10:31.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:31 on dist-test-slave-q46p"
08:10:31.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.019620 14310 fs_manager.cc:696] Time spent creating directory manager: real 0.006s	user 0.007s	sys 0.001s
08:10:31.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.025230 14329 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:31.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.026273 14310 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.026633 14310 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/wal
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67f6c9330d05405d8014fc5778fc8dc6"
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:31 on dist-test-slave-q46p"
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.026944 14310 fs_report.cc:389] FS layout report
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/wal
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/wal
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data/data
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:31.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:31.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.075263 14310 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:31.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.076612 14310 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:31.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.077041 14310 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.079505 14310 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:10:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.083528 14310 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:10:31.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.083721 14310 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
08:10:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.083915 14310 ts_tablet_manager.cc:610] Registered 0 tablets
08:10:31.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.084046 14310 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
08:10:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.229894 14310 rpc_server.cc:307] RPC server started. Bound to: 127.13.104.195:36319
08:10:31.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.229995 14441 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.104.195:36319 every 8 connection(s)
08:10:31.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.232383 14310 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/data/info.pb
08:10:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.235354 13731 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14310
08:10:31.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.235864 13731 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster6657655615915305137/ts-2/wal/instance
08:10:31.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.273758 14443 heartbeater.cc:344] Connected to a master server at 127.13.104.252:44239
08:10:31.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.274152 14442 heartbeater.cc:344] Connected to a master server at 127.13.104.253:45325
08:10:31.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.274294 14444 heartbeater.cc:344] Connected to a master server at 127.13.104.254:36695
08:10:31.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.274428 14443 heartbeater.cc:461] Registering TS with master...
08:10:31.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.274665 14444 heartbeater.cc:461] Registering TS with master...
08:10:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.275017 14442 heartbeater.cc:461] Registering TS with master...
08:10:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.275719 14444 heartbeater.cc:507] Master 127.13.104.254:36695 requested a full tablet report, sending...
08:10:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.275882 14442 heartbeater.cc:507] Master 127.13.104.253:45325 requested a full tablet report, sending...
08:10:31.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.275846 14443 heartbeater.cc:507] Master 127.13.104.252:44239 requested a full tablet report, sending...
08:10:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.278247 13930 ts_manager.cc:194] Registered new tserver with Master: 67f6c9330d05405d8014fc5778fc8dc6 (127.13.104.195:36319)
08:10:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.278879 13851 ts_manager.cc:194] Registered new tserver with Master: 67f6c9330d05405d8014fc5778fc8dc6 (127.13.104.195:36319)
08:10:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.279004 13776 ts_manager.cc:194] Registered new tserver with Master: 67f6c9330d05405d8014fc5778fc8dc6 (127.13.104.195:36319)
08:10:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.279675 13930 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.104.195:60911
08:10:31.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.297400 13731 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
08:10:31.307 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
08:10:31.308 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

08:10:31.309 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.13.104.254"
      port: 36695
    }
    bound_http_address {
      host: "127.13.104.254"
      port: 43101
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.13.104.253"
      port: 45325
    }
    bound_http_address {
      host: "127.13.104.253"
      port: 45015
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.13.104.252"
      port: 44239
    }
    bound_http_address {
      host: "127.13.104.252"
      port: 39725
    }
  }
}

08:10:31.309 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

08:10:31.310 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.13.104.193"
      port: 42603
    }
    bound_http_address {
      host: "127.13.104.193"
      port: 46099
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.13.104.194"
      port: 43919
    }
    bound_http_address {
      host: "127.13.104.194"
      port: 33171
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.13.104.195"
      port: 36319
    }
    bound_http_address {
      host: "127.13.104.195"
      port: 46073
    }
  }
}

08:10:31.310 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:10:31.311 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster4369099640883534331/krb5kdc/krb5cc
08:10:31.311 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
08:10:31.312 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4Hf2Nr/test-tmp
08:10:31.312 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:10:31.313 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4Hf2Nr/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:10:31.314 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 0
  enable_kerberos: true
  cluster_root: "/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745"
  extra_master_flags: "--webserver_require_spnego=false"
  extra_master_flags: "--authn_token_validity_seconds=10"
  extra_tserver_flags: "--webserver_require_spnego=false"
  hms_mode: NONE
  mini_kdc_options {
    ticket_lifetime: "10s"
    renew_lifetime: "20s"
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

08:10:31.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:31.638 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}

08:10:31.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.641582 14453 mini_oidc.cc:150] Starting JWKS server
08:10:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.644739 14453 webserver.cc:480] Webserver started at http://127.0.0.1:39761/ using document root <none> and password file <none>
08:10:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.645181 14453 mini_oidc.cc:186] Starting OIDC Discovery server
08:10:31.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:31.646704 14453 webserver.cc:480] Webserver started at http://127.0.0.1:33649/ using document root <none> and password file <none>
08:10:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:10:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/principal' for realm 'KRBTEST.COM',
08:10:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:10:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:31 dist-test-slave-q46p krb5kdc[14464](info): setting up network...
08:10:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:10:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:31 dist-test-slave-q46p krb5kdc[14464](info): set up 2 sockets
08:10:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:31 dist-test-slave-q46p krb5kdc[14464](info): commencing operation
08:10:31.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:10:32.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:32.283258 14443 heartbeater.cc:499] Master 127.13.104.252:44239 was elected leader, sending a full tablet report...
08:10:33.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:33.743468 14453 mini_kdc.cc:121] Time spent starting KDC: real 2.061s	user 0.004s	sys 0.015s
08:10:33.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:10:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:10:33.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:10:33.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:10:33.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:10:33.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:10:33.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:10:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:10:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:10:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:10:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/test-user.keytab.
08:10:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/test-user.keytab.
08:10:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:33 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973433, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
08:10:33.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:10:33Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:10:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:10:33Z Disabled control of system clock
08:10:33.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.29.126@KRBTEST.COM; defaulting to no policy
08:10:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:33.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.29.126@KRBTEST.COM" created.
08:10:33.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.29.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.29.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:33.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.29.126@KRBTEST.COM; defaulting to no policy
08:10:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:33.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.29.126@KRBTEST.COM" created.
08:10:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.29.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:33.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.29.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:33.961408 14453 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data
08:10:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:33.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/logs
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/info.pb
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.126:40021
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.126
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.126
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:33.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:33.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:33.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:10:33.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:10:34.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.267488 14482 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:34.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.268036 14482 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:34.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.268659 14482 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.302610 14482 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:10:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.302910 14482 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.303194 14482 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.303431 14482 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.303632 14482 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.303823 14482 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:34.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.303967 14482 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39761/jwks.json
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:34.347043 14482 master_runner.cc:386] Master server non-default flags:
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.126
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.126:40021
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:34.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/info.pb
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.126
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/logs
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:34.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:34.348407 14482 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:34.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:34.349476 14482 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:34.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.361056 14488 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:35.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:34.361590 14489 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:35.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:35.605643 14482 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.247s	user 0.477s	sys 0.762s
08:10:35.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:35.606180 14482 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.248s	user 0.477s	sys 0.762s
08:10:35.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.607663 14482 server_base.cc:1047] running on GCE node
08:10:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:35.609433 14495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:35.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.610728 14482 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.613256 14482 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:35.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.614637 14482 hybrid_clock.cc:648] HybridClock initialized: now 1756973435614604 us; error 46 us; skew 500 ppm
08:10:35.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.615530 14482 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:35.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:35 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973435, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.622375 14482 init.cc:377] Logged in from keytab as kudu/127.14.29.126@KRBTEST.COM (short username kudu)
08:10:35.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.628310 14482 webserver.cc:480] Webserver started at http://127.14.29.126:35685/ using document root <none> and password file <none>
08:10:35.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.629576 14482 fs_manager.cc:362] Metadata directory not provided
08:10:35.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.629859 14482 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:35.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.630532 14482 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.637434 14482 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/instance:
08:10:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1077fa5b7904acfae4faf66df68a255"
08:10:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:35 on dist-test-slave-q46p"
08:10:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.638549 14482 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal/instance:
08:10:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1077fa5b7904acfae4faf66df68a255"
08:10:35.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:35 on dist-test-slave-q46p"
08:10:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.646281 14482 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.000s
08:10:35.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.652452 14501 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.653604 14482 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.005s	sys 0.000s
08:10:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.653932 14482 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1077fa5b7904acfae4faf66df68a255"
08:10:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:35 on dist-test-slave-q46p"
08:10:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.654307 14482 fs_report.cc:389] FS layout report
08:10:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/data
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:35.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:35.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.725539 14482 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:35.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.734565 14482 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:35.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.735082 14482 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.811094 14482 rpc_server.cc:307] RPC server started. Bound to: 127.14.29.126:40021
08:10:35.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.811154 14552 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.29.126:40021 every 8 connection(s)
08:10:35.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.813830 14482 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/info.pb
08:10:35.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.819857 14553 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:10:35.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.822306 14453 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14482
08:10:35.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.822834 14453 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal/instance
08:10:35.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.830263 14553 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } has no permanent_uuid. Determining permanent_uuid...
08:10:35.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.29.125@KRBTEST.COM; defaulting to no policy
08:10:35.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:35.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.29.125@KRBTEST.COM" created.
08:10:35.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:35 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973435, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.126@KRBTEST.COM for kudu/127.14.29.126@KRBTEST.COM
08:10:35.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:35.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.29.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:35.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.29.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:35.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.871202 14553 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } has no permanent_uuid. Determining permanent_uuid...
08:10:35.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:35.874681 14503 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:35.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:35.886132 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.125:46777: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:35.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.29.125@KRBTEST.COM; defaulting to no policy
08:10:35.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:35.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.29.125@KRBTEST.COM" created.
08:10:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.29.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:35.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.29.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:35.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.916520 14453 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/logs
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/info.pb
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.125:46777
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.125
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.125
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:35.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:10:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:10:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:35.935631 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } attempt: 1
08:10:35.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:35.939849 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.125:46777: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:36.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:36.008216 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } attempt: 2
08:10:36.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.012807 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.125:46777: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:36.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:36.104192 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } attempt: 3
08:10:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.108438 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.125:46777: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.223795 14562 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:36.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.224426 14562 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:36.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.225138 14562 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:36.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:36.251849 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } attempt: 4
08:10:36.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.255728 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.125:46777: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:36.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.259801 14562 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:10:36.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.260066 14562 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:36.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.260488 14562 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:36.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.260826 14562 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:36.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.261106 14562 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:36.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.261382 14562 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:36.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.261591 14562 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39761/jwks.json
08:10:36.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:36.298728 14562 master_runner.cc:386] Master server non-default flags:
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.125
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.125:46777
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/info.pb
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.125
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/logs
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:36.300513 14562 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:36.301913 14562 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:36.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.313227 14568 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:36.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:36.555127 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } attempt: 5
08:10:36.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.562697 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.125:46777: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.110304 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } attempt: 6
08:10:37.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:37.122354 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.125:46777: Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:36.314208 14569 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:37.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:37.701297 14562 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.390s	user 0.574s	sys 0.804s
08:10:37.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:37.701709 14562 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.390s	user 0.574s	sys 0.804s
08:10:37.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.703799 14562 server_base.cc:1047] running on GCE node
08:10:37.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:37.705338 14577 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:37.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.706627 14562 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:37.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.709256 14562 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.711383 14562 hybrid_clock.cc:648] HybridClock initialized: now 1756973437711345 us; error 40 us; skew 500 ppm
08:10:37.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.712648 14562 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:37.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:37 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973437, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:37.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.723055 14562 init.cc:377] Logged in from keytab as kudu/127.14.29.125@KRBTEST.COM (short username kudu)
08:10:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.729678 14562 webserver.cc:480] Webserver started at http://127.14.29.125:34607/ using document root <none> and password file <none>
08:10:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.730962 14562 fs_manager.cc:362] Metadata directory not provided
08:10:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.731153 14562 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.731519 14562 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.736383 14562 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/instance:
08:10:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1025b7a337f54688b95f5e25b95b6238"
08:10:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:37 on dist-test-slave-q46p"
08:10:37.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.737608 14562 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal/instance:
08:10:37.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1025b7a337f54688b95f5e25b95b6238"
08:10:37.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:37 on dist-test-slave-q46p"
08:10:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.745380 14562 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.002s	sys 0.006s
08:10:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.751631 14583 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.752772 14562 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
08:10:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.753080 14562 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1025b7a337f54688b95f5e25b95b6238"
08:10:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:37 on dist-test-slave-q46p"
08:10:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.753371 14562 fs_report.cc:389] FS layout report
08:10:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:37.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/data
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:37.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:37.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.843981 14562 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:37.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.849530 14562 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:37.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.849920 14562 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:37.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.920840 14562 rpc_server.cc:307] RPC server started. Bound to: 127.14.29.125:46777
08:10:37.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.920910 14635 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.29.125:46777 every 8 connection(s)
08:10:37.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.924052 14562 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/info.pb
08:10:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.929342 14453 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14562
08:10:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.929872 14453 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal/instance
08:10:37.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.930079 14636 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:10:37.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.943454 14636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } has no permanent_uuid. Determining permanent_uuid...
08:10:37.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.14.29.124@KRBTEST.COM; defaulting to no policy
08:10:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.14.29.124@KRBTEST.COM" created.
08:10:37.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:37 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973437, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.125@KRBTEST.COM for kudu/127.14.29.126@KRBTEST.COM
08:10:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.29.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.14.29.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:37.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:37.983321 14636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } has no permanent_uuid. Determining permanent_uuid...
08:10:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.14.29.124@KRBTEST.COM; defaulting to no policy
08:10:37.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:37.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.14.29.124@KRBTEST.COM" created.
08:10:37.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:37 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973437, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.125@KRBTEST.COM for kudu/127.14.29.125@KRBTEST.COM
08:10:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.009727 14636 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } has no permanent_uuid. Determining permanent_uuid...
08:10:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.011972 14586 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:10:38.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.29.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:38.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.14.29.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab.
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.015847 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.015830 14453 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/logs
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/info.pb
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.124:36979
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.124
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.124
08:10:38.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:10:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:10:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.065302 14636 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 1
08:10:38.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.068964 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.137287 14636 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 2
08:10:38.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.141003 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.182943 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } attempt: 7
08:10:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:38 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973435, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.126@KRBTEST.COM for kudu/127.14.29.125@KRBTEST.COM
08:10:38.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.212373 14553 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } has no permanent_uuid. Determining permanent_uuid...
08:10:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.218191 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.232436 14636 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 3
08:10:38.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.235997 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.276690 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 1
08:10:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.281244 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.335628 14646 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.336153 14646 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.336848 14646 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.362769 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 2
08:10:38.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.366533 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.368942 14646 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:10:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.369151 14646 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.369421 14646 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.369623 14646 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.369836 14646 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.370024 14646 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.370168 14646 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39761/jwks.json
08:10:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.379426 14636 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 4
08:10:38.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.383724 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.410019 14646 master_runner.cc:386] Master server non-default flags:
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.124
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.124:36979
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/info.pb
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.124
08:10:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/logs
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.411413 14646 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:38.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.412408 14646 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:38.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.424881 14655 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:38.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.451988 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 3
08:10:38.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.454316 14438 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4f00 after lost signal to thread 14313
08:10:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.455669 14438 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac1a0 after lost signal to thread 14441
08:10:38.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.460621 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.601037 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 4
08:10:38.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.609087 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.683104 14636 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 5
08:10:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.688380 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:38.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:38.892731 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 5
08:10:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.900733 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:39.235814 14636 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 6
08:10:39.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:39.275851 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:39.454919 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 6
08:10:39.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:39.492038 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:40.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.337492 14636 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 7
08:10:40.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.357537 14636 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:39.825584 14653 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 14646
08:10:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.366550 14653 kernel_stack_watchdog.cc:198] Thread 14646 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
08:10:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:10:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:10:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:10:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:10:40.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:38.426064 14654 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.368799 14656 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1940 milliseconds
08:10:40.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.369449 14646 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.942s	user 0.001s	sys 0.000s
08:10:40.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.369800 14646 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.943s	user 0.001s	sys 0.000s
08:10:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.371181 14657 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.371133 14646 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:40.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.374770 14646 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:40.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.377591 14646 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:40.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.378955 14646 hybrid_clock.cc:648] HybridClock initialized: now 1756973440378913 us; error 34 us; skew 500 ppm
08:10:40.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.379823 14646 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:40.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973440, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.385426 14646 init.cc:377] Logged in from keytab as kudu/127.14.29.124@KRBTEST.COM (short username kudu)
08:10:40.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.392050 14646 webserver.cc:480] Webserver started at http://127.14.29.124:43117/ using document root <none> and password file <none>
08:10:40.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.392918 14646 fs_manager.cc:362] Metadata directory not provided
08:10:40.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.393126 14646 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:40.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.393594 14646 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
08:10:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.398129 14646 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/instance:
08:10:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d3428565b614fd3844bb1483c3176c1"
08:10:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:40 on dist-test-slave-q46p"
08:10:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.399572 14646 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal/instance:
08:10:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d3428565b614fd3844bb1483c3176c1"
08:10:40.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:40 on dist-test-slave-q46p"
08:10:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.407320 14646 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.007s	sys 0.001s
08:10:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.413367 14674 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.414716 14646 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.006s	sys 0.001s
08:10:40.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.415092 14646 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d3428565b614fd3844bb1483c3176c1"
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:40 on dist-test-slave-q46p"
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.415414 14646 fs_report.cc:389] FS layout report
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/data
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:40.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.475315 14646 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.483412 14646 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:40.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.484002 14646 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:40.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.525540 14553 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } attempt: 7
08:10:40.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.539558 14553 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.14.29.124:36979: Network error: Client connection negotiation failed: client connection to 127.14.29.124:36979: connect: Connection refused (error 111)
08:10:40.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.581945 14646 rpc_server.cc:307] RPC server started. Bound to: 127.14.29.124:36979
08:10:40.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.582023 14727 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.29.124:36979 every 8 connection(s)
08:10:40.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.585974 14646 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/info.pb
08:10:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.594904 14453 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14646
08:10:40.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.595368 14453 external_mini_cluster.cc:1442] Reading /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal/instance
08:10:40.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.596019 14728 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:10:40.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.611428 14728 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } has no permanent_uuid. Determining permanent_uuid...
08:10:40.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973433, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.29.124@KRBTEST.COM
08:10:40.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973440, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.124@KRBTEST.COM for kudu/127.14.29.126@KRBTEST.COM
08:10:40.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.665342 14728 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } has no permanent_uuid. Determining permanent_uuid...
08:10:40.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973433, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.29.125@KRBTEST.COM
08:10:40.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973440, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.124@KRBTEST.COM for kudu/127.14.29.125@KRBTEST.COM
08:10:40.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973433, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.29.126@KRBTEST.COM
08:10:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.697260 14728 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } has no permanent_uuid. Determining permanent_uuid...
08:10:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973440, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.124@KRBTEST.COM for kudu/127.14.29.124@KRBTEST.COM
08:10:40.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.711972 14453 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
08:10:40.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:40.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}

08:10:40.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
  env_vars {
    key: "KRB5CCNAME"
    value: "/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc"
  }
  env_vars {
    key: "KRB5_CONFIG"
    value: "/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5.conf"
  }
  env_vars {
    key: "KRB5_KDC_PROFILE"
    value: "/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kdc.conf"
  }
  env_vars {
    key: "KUDU_ENABLE_KRB5_REALM_FIX"
    value: "yes"
  }
}

08:10:40.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}

08:10:40.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.14.29.126"
      port: 40021
    }
    bound_http_address {
      host: "127.14.29.126"
      port: 35685
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.14.29.125"
      port: 46777
    }
    bound_http_address {
      host: "127.14.29.125"
      port: 34607
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.14.29.124"
      port: 36979
    }
    bound_http_address {
      host: "127.14.29.124"
      port: 43117
    }
  }
}

08:10:40.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}

08:10:40.726 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:40.726 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:10:40.726 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

08:10:40.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.732120 14728 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Bootstrap starting.
08:10:40.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.740034 14728 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Neither blocks nor log segments found. Creating new log.
08:10:40.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.741989 14728 log.cc:826] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Log is configured to *not* fsync() on all Append() calls
08:10:40.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.747087 14728 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: No bootstrap required, opened a new log
08:10:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:40 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973440, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
08:10:40.767 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:40.768 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc
08:10:40.769 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:10:40.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.770308 14728 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:40.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.771219 14728 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:10:40.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.771548 14728 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d3428565b614fd3844bb1483c3176c1, State: Initialized, Role: FOLLOWER
08:10:40.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.772243 14728 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:40.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.774614 14739 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } } }
08:10:40.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.775556 14739 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [sys.catalog]: This master's current role is: FOLLOWER
08:10:40.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.776170 14728 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [sys.catalog]: configured and running, proceeding with master startup.
08:10:40.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.805253 14750 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:40.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.805763 14750 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.807788 14646 master_runner.cc:186] Error getting master registration for 127.14.29.126:40021: OK, instance_id { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" instance_seqno: 1756973435788608 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:10:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:40.836086 14646 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:10:40.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:40.990664 14680 tablet.cc:2378] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:10:41.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:41.133208 14739 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:10:41.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:41.133812 14739 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:41.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:41.136802 14739 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:10:41.770 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.29.125:46777
08:10:41.771 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

08:10:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:41.772048 14453 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 14562
08:10:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:41.797112 14676 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.14.29.125:46777 (error 108)
08:10:41.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:41.800 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.29.126:40021
08:10:41.800 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

08:10:41.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:41.800460 14453 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 14482
08:10:41.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:41.807209 14676 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:41.810328 14750 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:41.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:41.810623 14750 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:41.828 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:41.828 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.14.29.124:36979
08:10:41.828 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

08:10:41.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:41.829150 14453 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 14646
08:10:41.854 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:41.854 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.29.125:46777
08:10:41.854 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 1
  }
}

08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:41.855718 14453 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/logs
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/info.pb
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.125:46777
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.125
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34607
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:41.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.125
08:10:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:10:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:10:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:10:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:41.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:10:41.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:10:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.157588 14755 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.158123 14755 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.158761 14755 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.190240 14755 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:10:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.190486 14755 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:42.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.190743 14755 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.190987 14755 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.191192 14755 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.191375 14755 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.191524 14755 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39761/jwks.json
08:10:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:42.230327 14755 master_runner.cc:386] Master server non-default flags:
08:10:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data
08:10:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.125
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.125:46777
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/info.pb
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.125
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34607
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/logs
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:42.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:42.231972 14755 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:42.233004 14755 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:42.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.244136 14762 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:43.649076 14761 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 14755
08:10:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.031174 14761 kernel_stack_watchdog.cc:198] Thread 14755 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
08:10:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:10:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:10:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:10:44.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:10:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:42.244786 14763 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.032111 14755 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.787s	user 0.666s	sys 1.068s
08:10:44.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.032689 14755 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.788s	user 0.666s	sys 1.069s
08:10:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.034078 14765 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:44.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.037904 14764 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1791 milliseconds
08:10:44.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.037937 14755 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:44.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.039233 14755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:44.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.041596 14755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:44.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.042970 14755 hybrid_clock.cc:648] HybridClock initialized: now 1756973444042918 us; error 50 us; skew 500 ppm
08:10:44.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.043812 14755 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:44.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:44 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973444, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:44.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.049964 14755 init.cc:377] Logged in from keytab as kudu/127.14.29.125@KRBTEST.COM (short username kudu)
08:10:44.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.054510 14755 webserver.cc:480] Webserver started at http://127.14.29.125:34607/ using document root <none> and password file <none>
08:10:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.055403 14755 fs_manager.cc:362] Metadata directory not provided
08:10:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.055616 14755 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.063649 14755 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.001s
08:10:44.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.067893 14773 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:44.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.068979 14755 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.001s	sys 0.003s
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.069271 14755 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1025b7a337f54688b95f5e25b95b6238"
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:37 on dist-test-slave-q46p"
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.071197 14755 fs_report.cc:389] FS layout report
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/data
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:44.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:44.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.131062 14755 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.140144 14755 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:44.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.140552 14755 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.216339 14755 rpc_server.cc:307] RPC server started. Bound to: 127.14.29.125:46777
08:10:44.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.216413 14824 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.29.125:46777 every 8 connection(s)
08:10:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.219259 14755 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/data/info.pb
08:10:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.228895 14453 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14755
08:10:44.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.229437 14825 sys_catalog.cc:263] Verifying existing consensus state
08:10:44.229 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.29.126:40021
08:10:44.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 0
  }
}

08:10:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20250904 08:10:44.230402 14825 master.cc:416] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:10:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.230774 14755 master.cc:561] Master@127.14.29.125:46777 shutting down...
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.231374 14453 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/logs
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/info.pb
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:44.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.126:40021
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.126
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35685
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.126
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:10:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:10:44.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.291263 14755 master.cc:583] Master@127.14.29.125:46777 shutdown complete.
08:10:44.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.293670 14755 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
08:10:44.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:10:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.547166 14827 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.547689 14827 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.548614 14827 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.580308 14827 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:10:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.580538 14827 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.580792 14827 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.580983 14827 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.581169 14827 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.581348 14827 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.581492 14827 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39761/jwks.json
08:10:44.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.617564 14827 master_runner.cc:386] Master server non-default flags:
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.126
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.126:40021
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/info.pb
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.126
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35685
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/logs
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.618760 14827 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.619803 14827 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:44.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.629402 14834 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:45.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:44.634908 14827 server_base.cc:1047] running on GCE node
08:10:45.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.634807 14837 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:45.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:44.631377 14835 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:45.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.834280 14827 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:45.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.836992 14827 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:45.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.838363 14827 hybrid_clock.cc:648] HybridClock initialized: now 1756973445838315 us; error 52 us; skew 500 ppm
08:10:45.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.839222 14827 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:45.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:45 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973445, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.845588 14827 init.cc:377] Logged in from keytab as kudu/127.14.29.126@KRBTEST.COM (short username kudu)
08:10:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.850762 14827 webserver.cc:480] Webserver started at http://127.14.29.126:35685/ using document root <none> and password file <none>
08:10:45.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.851692 14827 fs_manager.cc:362] Metadata directory not provided
08:10:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.851883 14827 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:45.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.859536 14827 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.004s	sys 0.004s
08:10:45.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.864018 14845 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.865056 14827 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.004s	sys 0.001s
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.865345 14827 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e1077fa5b7904acfae4faf66df68a255"
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:35 on dist-test-slave-q46p"
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.867244 14827 fs_report.cc:389] FS layout report
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/data
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:45.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.919718 14827 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.927677 14827 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:45.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:45.928439 14827 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.013244 14827 rpc_server.cc:307] RPC server started. Bound to: 127.14.29.126:40021
08:10:46.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.013319 14896 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.29.126:40021 every 8 connection(s)
08:10:46.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.015940 14827 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/data/info.pb
08:10:46.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.026273 14453 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14827
08:10:46.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:46.027 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.14.29.124:36979
08:10:46.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 2
  }
}

08:10:46.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.027158 14897 sys_catalog.cc:263] Verifying existing consensus state
08:10:46.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20250904 08:10:46.028100 14897 master.cc:416] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:10:46.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.028542 14827 master.cc:561] Master@127.14.29.126:40021 shutting down...
08:10:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.028725 14453 external_mini_cluster.cc:1366] Running /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu
08:10:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data
08:10:46.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/logs
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/info.pb
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.124:36979
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.124
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43117
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.124
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:10:46.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:10:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.089222 14827 master.cc:583] Master@127.14.29.126:40021 shutdown complete.
08:10:46.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.092360 14827 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
08:10:46.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
08:10:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.343050 14899 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:10:46.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.343605 14899 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:10:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.344338 14899 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:10:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.375696 14899 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:10:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.375909 14899 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:10:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.376340 14899 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:10:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.376667 14899 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:10:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.376969 14899 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:10:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.377254 14899 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:10:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.377467 14899 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39761/jwks.json
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.429728 14899 master_runner.cc:386] Master server non-default flags:
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.14.29.84:39227
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.14.29.126:40021,127.14.29.125:46777,127.14.29.124:36979
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/kudu.keytab
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.14.29.124
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.14.29.124:36979
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:10:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39761/jwks.json
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/info.pb
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.14.29.124
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43117
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/logs
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 07:43:23 UTC on 5fd53c4cbb9d
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7893
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:10:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.431002 14899 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:10:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:46.431965 14899 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:10:46.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.441773 14905 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.443550 14906 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:46.446699 14908 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:10:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:47.656425 14907 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:10:47.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.656554 14899 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:10:47.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.660900 14899 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:10:47.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.664093 14899 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:10:47.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.665521 14899 hybrid_clock.cc:648] HybridClock initialized: now 1756973447665481 us; error 59 us; skew 500 ppm
08:10:47.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.666253 14899 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:10:47.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:47 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973447, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:47.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.672205 14899 init.cc:377] Logged in from keytab as kudu/127.14.29.124@KRBTEST.COM (short username kudu)
08:10:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.677431 14899 webserver.cc:480] Webserver started at http://127.14.29.124:43117/ using document root <none> and password file <none>
08:10:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.678355 14899 fs_manager.cc:362] Metadata directory not provided
08:10:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.678606 14899 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:10:47.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.686585 14899 fs_manager.cc:714] Time spent opening directory manager: real 0.005s	user 0.006s	sys 0.000s
08:10:47.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.690917 14918 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
08:10:47.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.691874 14899 fs_manager.cc:730] Time spent opening block manager: real 0.003s	user 0.004s	sys 0.000s
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.692200 14899 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data,/tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d3428565b614fd3844bb1483c3176c1"
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 08:10:40 on dist-test-slave-q46p"
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.694025 14899 fs_report.cc:389] FS layout report
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/wal
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/data
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:10:47.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:10:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:10:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:10:47.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:10:47.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.742761 14899 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
08:10:47.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.748070 14899 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:10:47.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.748530 14899 kserver.cc:163] Server-wide thread pool size limit: 3276
08:10:47.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.822088 14899 rpc_server.cc:307] RPC server started. Bound to: 127.14.29.124:36979
08:10:47.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.822124 14969 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.14.29.124:36979 every 8 connection(s)
08:10:47.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.825920 14899 server_base.cc:1179] Dumped server information to /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/master-2/data/info.pb
08:10:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.828662 14453 external_mini_cluster.cc:1428] Started /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu as pid 14899
08:10:47.829 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
08:10:47.830 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:47.833 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:47.835 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.14.29.124:36979(127.14.29.124:36979)] connecting to peer
08:10:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.840175 14970 sys_catalog.cc:263] Verifying existing consensus state
08:10:47.845 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.846 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.847 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.849 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.14.29.124:36979(127.14.29.124:36979)] Successfully connected to peer
08:10:47.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.853368 14970 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Bootstrap starting.
08:10:47.854 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.854 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.855 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.860 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
08:10:47.860 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4Hf2Nr/test-tmp/mini-kudu-cluster2204209787550133745/krb5kdc/krb5cc
08:10:47.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:47 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973440, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:47.863 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:10:47.864 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
08:10:47.865 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.14.29.124
08:10:47.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:47 dist-test-slave-q46p krb5kdc[14464](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973440, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.14.29.124@KRBTEST.COM
08:10:47.894 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xcf094113, L:/127.0.0.1:56794 - R:/127.14.29.124:36979] using SASL/GSSAPI
08:10:47.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.896689 14970 log.cc:826] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Log is configured to *not* fsync() on all Append() calls
08:10:47.900 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.124:36979: Catalog manager is not initialized. State: Starting
08:10:47.900 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.902010 14970 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:10:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.902680 14970 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Bootstrap complete.
08:10:47.912 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:47.914 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:47.915 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.915 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.915 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.917 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.917 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.917 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.918 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.124:36979: Catalog manager is not initialized. State: Starting
08:10:47.918 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:47.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.923573 14970 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:47.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.925983 14970 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:10:47.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.926196 14970 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d3428565b614fd3844bb1483c3176c1, State: Initialized, Role: FOLLOWER
08:10:47.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.926826 14970 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:47.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.929294 14978 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } } }
08:10:47.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.929878 14978 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [sys.catalog]: This master's current role is: FOLLOWER
08:10:47.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.930657 14970 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [sys.catalog]: configured and running, proceeding with master startup.
08:10:47.931 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:47.932 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:47.933 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.933 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.933 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.936 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.936 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.936 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.939 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.124:36979: Catalog manager is not initialized. State: Starting
08:10:47.939 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:47.950 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:47.951 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:47.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.959 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.959 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:47.957855 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:47.958202 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:47.962 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.963 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.963 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.963 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:47.971 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:47.971 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:47.975 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.975 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.975 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:47.976 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.976 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.976 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:47.977 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.965011 14899 master_runner.cc:186] Error getting master registration for 127.14.29.126:40021: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111), 
08:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:47.990410 14899 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:10:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.003401 14924 tablet.cc:2378] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:10:48.010 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:48.011 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:48.013 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.013 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.013 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.015 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.015 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.015 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.017 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:48.051 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:48.053 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:48.053 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.053 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.053 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.055 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.056 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.056 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.057 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:48.111 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:48.112 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:48.113 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.115 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.115 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.117 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:48.190 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:48.191 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:48.193 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.193 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.195 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.195 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.195 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.197 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:48.255895 14978 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:10:48.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:48.256428 14978 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:48.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:48.259400 14978 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:10:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.262032 14920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:10:48.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.266266 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021): Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:10:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.266919 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:48.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:48.267225 14920 leader_election.cc:304] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d3428565b614fd3844bb1483c3176c1; no voters: 1025b7a337f54688b95f5e25b95b6238, e1077fa5b7904acfae4faf66df68a255
08:10:48.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:48.267848 14978 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:10:48.371 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:48.372 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:48.374 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.374 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.374 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:48.374 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:48.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:48.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.415614 14438 debug-util.cc:398] Leaking SignalData structure 0x7b08000b48a0 after lost signal to thread 14313
08:10:48.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.416525 14438 debug-util.cc:398] Leaking SignalData structure 0x7b08000b55a0 after lost signal to thread 14441
08:10:48.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.961360 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:48.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:48.961637 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:49.371 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:49.372 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:49.374 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:49.374 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:49.374 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:49.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:49.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:49.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:49.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:49.964781 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:49.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:49.965066 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:49.985854 14995 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:10:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:49.986256 14995 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:49.987800 14995 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:10:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:49.994074 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021): Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:10:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:49.994557 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:49.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:49.994911 14920 leader_election.cc:304] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d3428565b614fd3844bb1483c3176c1; no voters: 1025b7a337f54688b95f5e25b95b6238, e1077fa5b7904acfae4faf66df68a255
08:10:49.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:49.995474 14995 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:10:50.890 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:50.891 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:50.893 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:50.893 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:50.893 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:50.893 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:50.894 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:50.894 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:50.895 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:50.967957 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:50.968245 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:51.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:51.537020 15000 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:10:51.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:51.537384 15000 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:51.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:51.538635 15000 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:10:51.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:51.544253 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021): Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:10:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:51.544941 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:51.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:51.545270 14920 leader_election.cc:304] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d3428565b614fd3844bb1483c3176c1; no voters: 1025b7a337f54688b95f5e25b95b6238, e1077fa5b7904acfae4faf66df68a255
08:10:51.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:51.545887 15000 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:10:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:51.971145 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:51.971405 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:52.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:52.974097 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:52.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:52.974347 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:52.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:52.993165 14899 master_runner.cc:186] Error getting master registration for 127.14.29.126:40021: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111), 
08:10:52.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:52.993543 14899 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:10:53.371 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:53.372 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:53.375 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:53.375 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:53.375 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:53.375 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:53.375 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:53.375 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:53.377 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:53.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:53.830843 14157 debug-util.cc:398] Leaking SignalData structure 0x7b08000b90a0 after lost signal to thread 14030
08:10:53.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:53.831817 14157 debug-util.cc:398] Leaking SignalData structure 0x7b0800042440 after lost signal to thread 14160
08:10:53.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:53.977083 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:53.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:53.977334 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:54.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:54.980326 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:54.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:54.980589 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:55.984738 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:55.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:55.985033 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:56.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:56.066959 15008 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:10:56.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:56.067461 15008 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:56.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:56.068923 15008 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:10:56.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:56.071475 14920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111) [suppressed 5 similar messages]
08:10:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:56.074533 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021): Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:10:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:56.075167 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:56.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:56.075443 14920 leader_election.cc:304] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d3428565b614fd3844bb1483c3176c1; no voters: 1025b7a337f54688b95f5e25b95b6238, e1077fa5b7904acfae4faf66df68a255
08:10:56.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:56.076403 15008 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:10:56.431 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:56.431 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:56.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:56.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:56.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:56.433 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:56.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:56.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:56.436 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:56.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:56.543766 13807 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad8e0 after lost signal to thread 13744
08:10:56.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:56.544529 13807 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1840 after lost signal to thread 13810
08:10:56.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:56.988392 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:56.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:56.988706 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:57.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:10:57 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973457, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:10:57.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:57.675905 14914 init.cc:295] Successfully reacquired a new kerberos TGT
08:10:57.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:57.993111 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:57.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:57.993482 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:57.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:57.997120 14899 master_runner.cc:186] Error getting master registration for 127.14.29.126:40021: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111), 
08:10:57.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:57.997699 14899 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:10:58.771 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:10:58.771 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:10:58.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:58.773 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:58.773 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:10:58.773 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:58.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:58.774 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:10:58.775 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:10:58.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:58.996342 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:58.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:58.996594 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:10:59.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:59.686686 15017 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:10:59.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:59.687212 15017 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:10:59.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:59.689005 15017 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:10:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:59.694269 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021): Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:10:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:59.694725 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:10:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:59.695009 14920 leader_election.cc:304] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d3428565b614fd3844bb1483c3176c1; no voters: 1025b7a337f54688b95f5e25b95b6238, e1077fa5b7904acfae4faf66df68a255
08:10:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:10:59.695573 15017 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:10:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:59.999435 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:10:59.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:10:59.999760 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:01.002522 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:01.002787 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:01.111 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:11:01.112 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:11:01.112 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:01.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:01.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:01.113 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:01.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:01.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:01.115 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:11:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:01.444988 15024 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:11:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:01.445333 15024 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:11:01.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:01.446818 15024 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:11:01.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:01.448909 14920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111) [suppressed 3 similar messages]
08:11:01.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:01.451380 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021): Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:11:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:01.452082 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:11:01.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:01.452358 14920 leader_election.cc:304] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d3428565b614fd3844bb1483c3176c1; no voters: 1025b7a337f54688b95f5e25b95b6238, e1077fa5b7904acfae4faf66df68a255
08:11:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:01.452850 15024 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:11:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:02.005700 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:02.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:02.005959 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:03.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:03.000458 14899 master_runner.cc:186] Error getting master registration for 127.14.29.126:40021: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111), 
08:11:03.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:03.000821 14899 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:11:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:03.008814 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:03.009081 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:04.011997 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:04.012251 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:04.751 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:11:04.751 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:11:04.752 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:04.752 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:04.753 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:04.753 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:04.753 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:04.753 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:04.755 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:11:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:05.015082 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:05.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:05.015338 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:05.950 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:11:05.951 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:11:05.952 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:05.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:05.952 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:05.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:05.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:05.953 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:05.954 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:11:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:06.018093 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:06.018616 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:07.022346 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:07.022693 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:07.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 08:11:07 dist-test-slave-q46p krb5kdc[14464](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756973467, etypes {rep=17 tkt=17 ses=17}, kudu/127.14.29.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:11:07.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:07.678619 14914 init.cc:295] Successfully reacquired a new kerberos TGT
08:11:08.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:08.004719 14899 master_runner.cc:186] Error getting master registration for 127.14.29.126:40021: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111), 
08:11:08.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:08.005110 14899 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:11:08.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:08.025600 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:08.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:08.025928 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:08.610 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:11:08.611 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:11:08.612 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:08.612 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:08.612 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:08.613 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:08.613 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:08.613 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:08.615 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:11:09.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:09.028831 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:09.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:09.029094 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:10.031908 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:10.032179 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:10.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:10.434363 14298 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba3a0 after lost signal to thread 14173
08:11:10.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:10.435261 14298 debug-util.cc:398] Leaking SignalData structure 0x7b08000acf40 after lost signal to thread 14301
08:11:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:11.034826 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:11.035132 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:11.730 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:11:11.731 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:11:11.732 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:11.732 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:11.733 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:11.733 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:11.733 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:11.733 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:11.735 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:11:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:12.038122 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:12.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:12.038470 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:12.891 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:11:12.892 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:11:12.892 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:12.892 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:12.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:12.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:12.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:12.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:12.894 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:11:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:13.007643 14899 master_runner.cc:186] Error getting master registration for 127.14.29.126:40021: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111), 
08:11:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:13.008061 14899 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:13.041366 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:13.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:13.041651 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:13.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:13.684407 15043 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:11:13.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:13.684850 15043 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e1077fa5b7904acfae4faf66df68a255" member_type: VOTER last_known_addr { host: "127.14.29.126" port: 40021 } } peers { permanent_uuid: "1025b7a337f54688b95f5e25b95b6238" member_type: VOTER last_known_addr { host: "127.14.29.125" port: 46777 } } peers { permanent_uuid: "3d3428565b614fd3844bb1483c3176c1" member_type: VOTER last_known_addr { host: "127.14.29.124" port: 36979 } }
08:11:13.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:13.686373 15043 leader_election.cc:290] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021), 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777)
08:11:13.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:13.688822 14920 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111) [suppressed 1 similar messages]
08:11:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:13.692747 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer e1077fa5b7904acfae4faf66df68a255 (127.14.29.126:40021): Network error: Client connection negotiation failed: client connection to 127.14.29.126:40021: connect: Connection refused (error 111)
08:11:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:13.693231 14920 leader_election.cc:336] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 1025b7a337f54688b95f5e25b95b6238 (127.14.29.125:46777): Network error: Client connection negotiation failed: client connection to 127.14.29.125:46777: connect: Connection refused (error 111)
08:11:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:13.693470 14920 leader_election.cc:304] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d3428565b614fd3844bb1483c3176c1; no voters: 1025b7a337f54688b95f5e25b95b6238, e1077fa5b7904acfae4faf66df68a255
08:11:13.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:13.693982 15043 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
08:11:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:14.044689 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:14.044936 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:15.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:15.047613 14989 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3d3428565b614fd3844bb1483c3176c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:11:15.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:15.047888 14989 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:11:15.431 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.125:46777(127.14.29.125:46777)] connecting to peer
08:11:15.432 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.14.29.126:40021(127.14.29.126:40021)] connecting to peer
08:11:15.432 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:15.432 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.14.29.125:46777(127.14.29.125:46777)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:15.432 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.125:46777: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777
08:11:15.433 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:15.433 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.126:40021(127.14.29.126:40021)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:15.433 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.14.29.126:40021: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
08:11:15.435 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979; will retry
08:11:15.437 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27605), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [18ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [26ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [70ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [71ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [82ms] refreshing cache from master, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [86ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [88ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [89ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [90ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [104ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [107ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [110ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [110ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [130ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [133ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [134ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [134ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [146ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [147ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [148ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [148ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [184ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [186ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [187ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [221ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [224ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [227ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [228ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [281ms] refreshing cache from master, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [284ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [286ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [288ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [288ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [361ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [364ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [366ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [368ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [368ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1541ms] refreshing cache from master, [1542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [1545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [1546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [1547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3061ms] refreshing cache from master, [3061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [3062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [3063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [3064ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [3065ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3066ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [3067ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [5541ms] refreshing cache from master, [5542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [5543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [5544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [5546ms] too many traces: truncated at 100 traces, deferred=Deferred@1927022488(state=PENDING, result=null, callback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445), errback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445)))
08:11:15.440 [DEBUG - main] (Connection.java:575) [peer master-127.14.29.124:36979(127.14.29.124:36979)] disconnecting while in state READY
08:11:15.440 [DEBUG - main] (Connection.java:575) [peer master-127.14.29.125:46777(127.14.29.125:46777)] disconnecting while in state TERMINATED
08:11:15.440 [DEBUG - main] (Connection.java:575) [peer master-127.14.29.126:40021(127.14.29.126:40021)] disconnecting while in state TERMINATED
08:11:15.441 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:11:15.442 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.14.29.124:36979(127.14.29.124:36979)] handling channelInactive
08:11:15.442 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.14.29.124:36979(127.14.29.124:36979)] cleaning up while in state READY due to: connection closed
08:11:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:15.444118 14453 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 14899
08:11:16.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:11:16Z chronyd exiting
08:11:17.611 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:11:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:17.613195 13731 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 14010
08:11:17.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:17.639207 13731 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 14165
08:11:17.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 08:11:17.638543 13760 connection.cc:537] server connection from 127.13.104.193:37389 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:11:17.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:17.667727 13731 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 14310
08:11:17.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:17.693181 13731 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 13743
08:11:17.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:17.727257 13731 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 13813
08:11:17.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 08:11:17.757261 13731 external_mini_cluster.cc:1658] Killing /tmp/dist-test-task4Hf2Nr/build/tsan/bin/kudu with pid 13888
08:11:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T08:11:18Z chronyd exiting
08:11:19.907 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27605), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [18ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [26ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [70ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [71ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [82ms] refreshing cache from master, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [86ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [88ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [89ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [90ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [104ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [107ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [110ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [110ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [130ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [133ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [134ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [134ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [146ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [147ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [148ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [148ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [184ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [186ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [187ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [221ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [224ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [227ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [228ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [281ms] refreshing cache from master, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [284ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [286ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [288ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [288ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [361ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [364ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [366ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [368ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [368ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1541ms] refreshing cache from master, [1542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [1545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [1546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [1547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3061ms] refreshing cache from master, [3061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [3062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [3063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [3064ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [3065ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3066ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [3067ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [5541ms] refreshing cache from master, [5542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [5543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [5544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [5546ms] too many traces: truncated at 100 traces, deferred=Deferred@1927022488(state=PENDING, result=null, callback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445), errback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
	at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.14.29.126:40021
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27605), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [18ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [26ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [70ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [71ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [82ms] refreshing cache from master, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [86ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [88ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [89ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [90ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [104ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [107ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [110ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [110ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [130ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [133ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [134ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [134ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [146ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [147ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [148ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [148ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [184ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [186ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [187ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [221ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [224ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [227ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [228ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [281ms] refreshing cache from master, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [284ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [286ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [288ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [288ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [361ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [364ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [366ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [368ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [368ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1541ms] refreshing cache from master, [1542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [1545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [1546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [1547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3061ms] refreshing cache from master, [3061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [3062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [3063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [3064ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [3065ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3066ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [3067ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [5541ms] refreshing cache from master, [5542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [5543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [5544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [5546ms] too many traces: truncated at 100 traces, deferred=Deferred@1927022488(state=PENDING, result=null, callback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445), errback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
	at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.14.29.126:40021
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

08:11:20.116 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: giving up after 1 attempts
08:11:20.116 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27605), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [18ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [26ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [70ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [71ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [82ms] refreshing cache from master, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [86ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [88ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [89ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [90ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [104ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [107ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [110ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [110ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [130ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [133ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [134ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [134ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [146ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [147ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [148ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [148ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [184ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [186ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [187ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [221ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [224ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [227ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [228ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [281ms] refreshing cache from master, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [284ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [286ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [288ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [288ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [361ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [364ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [366ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [368ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [368ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1541ms] refreshing cache from master, [1542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [1545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [1546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [1547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3061ms] refreshing cache from master, [3061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [3062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [3063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [3064ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [3065ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3066ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [3067ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [5541ms] refreshing cache from master, [5542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [5543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [5544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [5546ms] too many traces: truncated at 100 traces, deferred=Deferred@1927022488(state=PENDING, result=null, callback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445), errback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
	at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.14.29.126:40021
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_275]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_275]
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=27605), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [6ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [18ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [26ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [70ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [71ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [82ms] refreshing cache from master, [83ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [86ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [86ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [88ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [89ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [90ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [104ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [107ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [110ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: Service unavailable: Catalog manager is not initialized. State: Starting (error 0), [110ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021,org.apache.kudu.client.RecoverableException: Catalog manager is not initialized. State: Starting, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [122ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [130ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [133ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [134ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [134ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [141ms] refreshing cache from master, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [146ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [147ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [148ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [148ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [183ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [184ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [186ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [187ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [188ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [221ms] refreshing cache from master, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [223ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [224ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [224ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [227ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [228ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [228ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [281ms] refreshing cache from master, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [282ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [284ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [284ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [286ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [288ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [288ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [361ms] refreshing cache from master, [361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [364ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [366ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [368ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [368ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [542ms] refreshing cache from master, [542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1541ms] refreshing cache from master, [1542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [1543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [1545ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [1546ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [1547ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [1547ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3061ms] refreshing cache from master, [3061ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [3062ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [3063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [3064ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.125:46777: Network error: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777, [3065ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.126:40021: Network error: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [3066ms] Sub RPC ConnectToMaster: received response from server master-127.14.29.124:36979: OK, [3067ms] delaying RPC due to: Service unavailable: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021, [5541ms] refreshing cache from master, [5542ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.125:46777, [5543ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.126:40021, [5544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.14.29.124:36979, [5546ms] too many traces: truncated at 100 traces, deferred=Deferred@1927022488(state=PENDING, result=null, callback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445), errback=(continuation of Deferred@1623157103 after retry RPC after error@1372449550) -> (continuation of Deferred@2013402607 after retry RPC after error@1785740720) -> (continuation of Deferred@1906296192 after retry RPC after error@1337251110) -> (continuation of Deferred@661724242 after retry RPC after error@1283930289) -> (continuation of Deferred@1127958372 after retry RPC after error@718256064) -> (continuation of Deferred@99859783 after retry RPC after error@1891913720) -> (continuation of Deferred@1767712483 after retry RPC after error@1056701887) -> (continuation of Deferred@645555874 after retry RPC after error@1955808794) -> (continuation of Deferred@1962478677 after retry RPC after error@353239408) -> (continuation of Deferred@1614035461 after retry RPC after error@251200399) -> (continuation of Deferred@845036961 after retry RPC after error@1463729947) -> (continuation of Deferred@370885611 after retry RPC after error@1345900508) -> (continuation of Deferred@323502927 after retry RPC after error@825028698) -> (continuation of Deferred@2103765391 after retry RPC after error@1806995408) -> (continuation of Deferred@976747474 after retry RPC after error@1825534702) -> (continuation of Deferred@918406761 after retry RPC after error@2033714529) -> (continuation of Deferred@2085914879 after retry RPC after error@1796984217) -> (continuation of Deferred@860737565 after retry RPC after error@182962200) -> (continuation of Deferred@1387051723 after retry RPC after error@1743704578) -> (continuation of Deferred@981351386 after retry RPC after error@242058117) -> (continuation of Deferred@966793825 after retry RPC after error@676311445)))
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
	at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.lang.Thread.run(Thread.java:748)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
		at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
		at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
		at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
		at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
		at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
		at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
		at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
		at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
		at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
		at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
		at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
		... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.14.29.125:46777,127.14.29.126:40021,127.14.29.124:36979) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.125:46777(127.14.29.125:46777): Connection refused: /127.14.29.125:46777,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
	at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
	at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
	at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
	at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1503)
	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1366)
	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1415)
	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:530)
	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:469)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.14.29.126:40021(127.14.29.126:40021): Connection refused: /127.14.29.126:40021
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
	at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
	at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
	at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
	at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
	at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
	at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
	at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.14.29.126:40021
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
	at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
	at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)