Diagnosed failure

Unable to diagnose

Full log

09:03:24.549 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
09:03:24.562 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskhl46EI/test-tmp
09:03:24.570 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:03:24.574 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskhl46EI/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:03:24.777 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: false
  cluster_root: "/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071"
  hms_mode: NONE
  mini_kdc_options {
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

09:03:25.009 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
09:03:25.010 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}

09:03:25.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.016409  8895 mini_oidc.cc:150] Starting JWKS server
09:03:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.021389  8895 webserver.cc:577] Webserver started at http://127.0.0.1:38299/ using document root <none> and password file <none>
09:03:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.022164  8895 mini_oidc.cc:186] Starting OIDC Discovery server
09:03:25.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.024474  8895 webserver.cc:577] Webserver started at http://127.0.0.1:44975/ using document root <none> and password file <none>
09:03:25.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T09:03:25Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:03:25.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T09:03:25Z Disabled control of system clock
09:03:25.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.088150  8895 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/wal
09:03:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data
09:03:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:25.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:25.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/logs
09:03:25.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data/info.pb
09:03:25.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:25.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:25.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:25.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:25.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:25.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:25.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:25.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.254:35127
09:03:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.254
09:03:25.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:25.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:25.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:25.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:25.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:25.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38299/jwks.json with env {}
09:03:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.404059  8907 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.404639  8907 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:25.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.405340  8907 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.443956  8907 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.444268  8907 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.444473  8907 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:25.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.444671  8907 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.444999  8907 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:25.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.445209  8907 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38299/jwks.json
09:03:25.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.484030  8907 master_runner.cc:387] Master server non-default flags:
09:03:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/logs
09:03:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:25.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data
09:03:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/wal
09:03:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.254:35127
09:03:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38299/jwks.json
09:03:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data/info.pb
09:03:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.254
09:03:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:25.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:25.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:25.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.485484  8907 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:25.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:25.487423  8907 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:25.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.500586  8913 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:26.904553  8912 debug-util.cc:398] Leaking SignalData structure 0x7b0800029240 after lost signal to thread 8907
09:03:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.334619  8912 kernel_stack_watchdog.cc:198] Thread 8907 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 401ms:
09:03:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:25.501874  8914 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.335919  8907 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.836s	user 0.707s	sys 1.126s
09:03:27.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.336303  8907 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.837s	user 0.707s	sys 1.126s
09:03:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.339633  8915 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1834 milliseconds
09:03:27.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.341271  8916 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:27.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.341320  8907 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:27.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.342861  8907 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:27.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.345654  8907 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:27.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.347062  8907 hybrid_clock.cc:648] HybridClock initialized: now 1794992607347026 us; error 48 us; skew 500 ppm
09:03:27.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.348188  8907 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:27.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.355187  8907 webserver.cc:577] Webserver started at http://127.8.175.254:38889/ using document root <none> and password file <none>
09:03:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.356600  8907 fs_manager.cc:362] Metadata directory not provided
09:03:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.356916  8907 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:27.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.357604  8907 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:27.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.362921  8907 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data/instance:
09:03:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12e9ccf2635a49f3bb38115f14251af9"
09:03:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:27 on dist-test-slave-mh3s"
09:03:27.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.364579  8907 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/wal/instance:
09:03:27.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12e9ccf2635a49f3bb38115f14251af9"
09:03:27.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:27 on dist-test-slave-mh3s"
09:03:27.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.373914  8907 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.009s	sys 0.001s
09:03:27.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.380977  8924 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:27.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.382627  8907 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.007s	sys 0.000s
09:03:27.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.383085  8907 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/wal
09:03:27.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "12e9ccf2635a49f3bb38115f14251af9"
09:03:27.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:27 on dist-test-slave-mh3s"
09:03:27.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.383693  8907 fs_report.cc:389] FS layout report
09:03:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/wal
09:03:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/wal
09:03:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data/data
09:03:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:27.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:27.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.456842  8907 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:27.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.458766  8907 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:27.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.459390  8907 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.537582  8907 rpc_server.cc:307] RPC server started. Bound to: 127.8.175.254:35127
09:03:27.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.537642  8975 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.175.254:35127 every 8 connection(s)
09:03:27.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.540654  8907 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/data/info.pb
09:03:27.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.543027  8895 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 8907
09:03:27.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.543681  8895 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-0/wal/instance
09:03:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.547773  8976 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.547875  8895 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/wal
09:03:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data
09:03:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:27.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/logs
09:03:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data/info.pb
09:03:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:27.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:27.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.253:42151
09:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.253
09:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:27.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:27.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38299/jwks.json with env {}
09:03:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.564085  8976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } has no permanent_uuid. Determining permanent_uuid...
09:03:27.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.593209  8976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } has no permanent_uuid. Determining permanent_uuid...
09:03:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.596280  8926 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:27.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.601046  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.253:42151: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.650645  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } attempt: 1
09:03:27.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.655994  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.253:42151: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:27.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.724459  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } attempt: 2
09:03:27.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.729836  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.253:42151: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.821286  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } attempt: 3
09:03:27.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.826642  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.253:42151: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:27.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.886974  8978 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.887707  8978 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:27.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.888588  8978 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:27.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.922183  8978 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.922631  8978 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.922963  8978 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:27.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.923333  8978 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:27.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.923754  8978 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:27.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.924045  8978 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38299/jwks.json
09:03:27.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.963611  8978 master_runner.cc:387] Master server non-default flags:
09:03:27.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/logs
09:03:27.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:27.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:27.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:27.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:27.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:27.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:27.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data
09:03:27.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/wal
09:03:27.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:27.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:27.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:27.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.253:42151
09:03:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38299/jwks.json
09:03:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data/info.pb
09:03:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.253
09:03:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:27.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:27.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:27.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:27.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:27.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.965477  8978 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:27.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.967985  8978 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:27.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:27.970141  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } attempt: 4
09:03:27.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.975855  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.253:42151: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:27.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.983233  8987 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:28.275293  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } attempt: 5
09:03:28.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:28.291683  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.253:42151: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:28.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:28.839264  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } attempt: 6
09:03:28.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:28.869302  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.253:42151: Network error: Client connection negotiation failed: client connection to 127.8.175.253:42151: connect: Connection refused (error 111)
09:03:29.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.386848  8986 debug-util.cc:398] Leaking SignalData structure 0x7b0800029240 after lost signal to thread 8978
09:03:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:27.983644  8988 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.434828  8978 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.452s	user 0.428s	sys 1.017s
09:03:29.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.435720  8978 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.453s	user 0.428s	sys 1.018s
09:03:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.437523  8990 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:29.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.440009  8989 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1451 milliseconds
09:03:29.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.440062  8978 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.441613  8978 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.444141  8978 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:29.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.445571  8978 hybrid_clock.cc:648] HybridClock initialized: now 1794992609445522 us; error 65 us; skew 500 ppm
09:03:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.446605  8978 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.454200  8978 webserver.cc:577] Webserver started at http://127.8.175.253:40411/ using document root <none> and password file <none>
09:03:29.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.455375  8978 fs_manager.cc:362] Metadata directory not provided
09:03:29.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.455729  8978 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:29.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.456276  8978 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:29.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.460968  8978 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data/instance:
09:03:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db2c29d9dbf6476785737f01269e1b8a"
09:03:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:29 on dist-test-slave-mh3s"
09:03:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.462343  8978 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/wal/instance:
09:03:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db2c29d9dbf6476785737f01269e1b8a"
09:03:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:29 on dist-test-slave-mh3s"
09:03:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.470434  8978 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.006s	sys 0.002s
09:03:29.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.476739  8999 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.478267  8978 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.007s	sys 0.000s
09:03:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.478562  8978 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/wal
09:03:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db2c29d9dbf6476785737f01269e1b8a"
09:03:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:29 on dist-test-slave-mh3s"
09:03:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.478865  8978 fs_report.cc:389] FS layout report
09:03:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/wal
09:03:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/wal
09:03:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data/data
09:03:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.540954  8978 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:29.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.542973  8978 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.543579  8978 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.622978  8978 rpc_server.cc:307] RPC server started. Bound to: 127.8.175.253:42151
09:03:29.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.623083  9051 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.175.253:42151 every 8 connection(s)
09:03:29.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.626463  8978 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/data/info.pb
09:03:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.635810  9052 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:29.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.636461  8895 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 8978
09:03:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.637126  8895 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-1/wal/instance
09:03:29.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.640358  8895 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/wal
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/logs
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data/info.pb
09:03:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.252:45213
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.252
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38299/jwks.json with env {}
09:03:29.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.651361  9052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } has no permanent_uuid. Determining permanent_uuid...
09:03:29.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.674724  9052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } has no permanent_uuid. Determining permanent_uuid...
09:03:29.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.691358  9052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
09:03:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.694099  9003 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:29.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.699034  9052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:29.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.748679  9052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 1
09:03:29.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.754439  9052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:29.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.822903  9052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 2
09:03:29.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.827515  9052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:29.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.918941  9052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 3
09:03:29.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.923270  9052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:29.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.929914  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } attempt: 7
09:03:29.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:29.941928  8976 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
09:03:29.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.949200  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:29.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.985289  9054 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.985864  9054 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:29.986439  9054 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:30.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.007688  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 1
09:03:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.012310  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:30.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.022446  9054 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.022756  9054 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.022960  9054 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.023257  9054 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.023502  9054 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:30.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.023659  9054 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:38299/jwks.json
09:03:30.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.066709  9052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 4
09:03:30.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.069446  9054 master_runner.cc:387] Master server non-default flags:
09:03:30.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/logs
09:03:30.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:30.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:30.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data
09:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/wal
09:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:30.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.252:45213
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38299/jwks.json
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data/info.pb
09:03:30.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.252
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:30.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.071676  9054 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.072279  9052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:30.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.074143  9054 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:30.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.088249  9066 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:30.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.093822  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 2
09:03:30.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.104439  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:30.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.190033  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 3
09:03:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.199046  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:30.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.339639  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 4
09:03:30.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.348774  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.371707  9052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 5
09:03:30.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.380752  9052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:30.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.632331  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 5
09:03:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.643411  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:30.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:30.928319  9052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 6
09:03:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.958940  9052 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:31.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.196003  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 6
09:03:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.221011  8976 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.8.175.252:45213: Network error: Client connection negotiation failed: client connection to 127.8.175.252:45213: connect: Connection refused (error 111)
09:03:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.630352  9054 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.542s	user 0.543s	sys 0.988s
09:03:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.491284  9065 debug-util.cc:398] Leaking SignalData structure 0x7b0800029240 after lost signal to thread 9054
09:03:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.632875  9054 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.545s	user 0.543s	sys 0.988s
09:03:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.633589  9068 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1543 milliseconds
09:03:31.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:30.089589  9067 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:31.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.636729  9069 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:31.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.637164  9054 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:31.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.640717  9054 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:31.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.643878  9054 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.645325  9054 hybrid_clock.cc:648] HybridClock initialized: now 1794992611645294 us; error 50 us; skew 500 ppm
09:03:31.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.646363  9054 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.653167  9054 webserver.cc:577] Webserver started at http://127.8.175.252:34341/ using document root <none> and password file <none>
09:03:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.654304  9054 fs_manager.cc:362] Metadata directory not provided
09:03:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.654553  9054 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:31.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.655089  9054 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.659531  9054 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data/instance:
09:03:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb05a59f29a6477394f9db9f0aaa1699"
09:03:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:31 on dist-test-slave-mh3s"
09:03:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.660823  9054 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/wal/instance:
09:03:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb05a59f29a6477394f9db9f0aaa1699"
09:03:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:31 on dist-test-slave-mh3s"
09:03:31.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.668864  9054 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.009s	sys 0.001s
09:03:31.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.674973  9080 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:31.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.676157  9054 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.002s
09:03:31.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.676530  9054 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/wal
09:03:31.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cb05a59f29a6477394f9db9f0aaa1699"
09:03:31.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:31 on dist-test-slave-mh3s"
09:03:31.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.676954  9054 fs_report.cc:389] FS layout report
09:03:31.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:31.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/wal
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/wal
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data/data
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:31.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:31.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:31.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:31.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.736342  9054 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:31.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.738034  9054 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:31.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.738634  9054 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.816972  9054 rpc_server.cc:307] RPC server started. Bound to: 127.8.175.252:45213
09:03:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.817057  9131 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.175.252:45213 every 8 connection(s)
09:03:31.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.820155  9054 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/data/info.pb
09:03:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.820974  8895 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9054
09:03:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.821452  8895 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/master-2/wal/instance
09:03:31.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.830969  9132 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.850600  9132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } has no permanent_uuid. Determining permanent_uuid...
09:03:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.877576  9132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } has no permanent_uuid. Determining permanent_uuid...
09:03:31.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.888873  9132 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } has no permanent_uuid. Determining permanent_uuid...
09:03:31.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.905277  9132 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699: Bootstrap starting.
09:03:31.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.912561  9132 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699: Neither blocks nor log segments found. Creating new log.
09:03:31.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.914958  9132 log.cc:826] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699: Log is configured to *not* fsync() on all Append() calls
09:03:31.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.921361  9132 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699: No bootstrap required, opened a new log
09:03:31.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.943864  9132 raft_consensus.cc:360] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.944557  9132 raft_consensus.cc:386] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:31.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.944826  9132 raft_consensus.cc:748] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cb05a59f29a6477394f9db9f0aaa1699, State: Initialized, Role: FOLLOWER
09:03:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.945765  9132 consensus_queue.cc:261] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.948685  9141 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.949270  9141 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: This master's current role is: FOLLOWER
09:03:31.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.950340  9132 sys_catalog.cc:565] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: configured and running, proceeding with master startup.
09:03:31.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.981506  9152 catalog_manager.cc:1609] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:31.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.981894  9152 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:31.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:31.994364  9086 tablet.cc:2441] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:31.975471  9054 master_runner.cc:187] Error getting master registration for 127.8.175.254:35127: OK, instance_id { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" instance_seqno: 1794992607517025 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:32.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.000355  9054 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:32.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.019616  9052 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 7
09:03:32.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.043081  9052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: Bootstrap starting.
09:03:32.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.053030  9052 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: Neither blocks nor log segments found. Creating new log.
09:03:32.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.055568  9052 log.cc:826] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: Log is configured to *not* fsync() on all Append() calls
09:03:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.062114  9052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: No bootstrap required, opened a new log
09:03:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.093451  9052 raft_consensus.cc:360] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:32.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.094205  9141 raft_consensus.cc:499] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:32.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.094434  9052 raft_consensus.cc:386] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:32.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.094727  9052 raft_consensus.cc:748] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db2c29d9dbf6476785737f01269e1b8a, State: Initialized, Role: FOLLOWER
09:03:32.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.094790  9141 raft_consensus.cc:521] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:32.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.095746  9052 consensus_queue.cc:261] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:32.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.098812  9141 leader_election.cc:290] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 12e9ccf2635a49f3bb38115f14251af9 (127.8.175.254:35127), db2c29d9dbf6476785737f01269e1b8a (127.8.175.253:42151)
09:03:32.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.100270  9156 sys_catalog.cc:455] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.101104  9156 sys_catalog.cc:458] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [sys.catalog]: This master's current role is: FOLLOWER
09:03:32.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.103757  9027 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cb05a59f29a6477394f9db9f0aaa1699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db2c29d9dbf6476785737f01269e1b8a" is_pre_election: true
09:03:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.104133  9052 sys_catalog.cc:565] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [sys.catalog]: configured and running, proceeding with master startup.
09:03:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.104594  9027 raft_consensus.cc:2490] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cb05a59f29a6477394f9db9f0aaa1699 in term 0.
09:03:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.106326  9082 leader_election.cc:304] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cb05a59f29a6477394f9db9f0aaa1699, db2c29d9dbf6476785737f01269e1b8a; no voters: 
09:03:32.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.107314  9141 raft_consensus.cc:2826] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.107662  9141 raft_consensus.cc:499] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:32.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.108016  9141 raft_consensus.cc:3096] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 0 FOLLOWER]: Advancing to term 1
09:03:32.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.118587  9141 raft_consensus.cc:521] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.122234  9141 leader_election.cc:290] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [CANDIDATE]: Term 1 election: Requested vote from peers 12e9ccf2635a49f3bb38115f14251af9 (127.8.175.254:35127), db2c29d9dbf6476785737f01269e1b8a (127.8.175.253:42151)
09:03:32.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.124397  9027 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cb05a59f29a6477394f9db9f0aaa1699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db2c29d9dbf6476785737f01269e1b8a"
09:03:32.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.125005  9027 raft_consensus.cc:3096] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [term 0 FOLLOWER]: Advancing to term 1
09:03:32.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.138335  9027 raft_consensus.cc:2490] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cb05a59f29a6477394f9db9f0aaa1699 in term 1.
09:03:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.141209  9082 leader_election.cc:304] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cb05a59f29a6477394f9db9f0aaa1699, db2c29d9dbf6476785737f01269e1b8a; no voters: 
09:03:32.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.142198  9141 raft_consensus.cc:2826] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 1 FOLLOWER]: Leader election won for term 1
09:03:32.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.156454  9141 raft_consensus.cc:705] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [term 1 LEADER]: Becoming Leader. State: Replica: cb05a59f29a6477394f9db9f0aaa1699, State: Running, Role: LEADER
09:03:32.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.158116  9141 consensus_queue.cc:238] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:32.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.169157  9165 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cb05a59f29a6477394f9db9f0aaa1699. Latest consensus state: current_term: 1 leader_uuid: "cb05a59f29a6477394f9db9f0aaa1699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.170526  9165 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: This master's current role is: LEADER
09:03:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.174089  9168 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:03:32.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.178072  9170 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
09:03:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.185266  9165 consensus_queue.cc:1049] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:03:32.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.194881  9170 catalog_manager.cc:1535] Initializing Kudu cluster ID...
09:03:32.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.204701  9156 sys_catalog.cc:455] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [sys.catalog]: SysCatalogTable state changed. Reason: New leader cb05a59f29a6477394f9db9f0aaa1699. Latest consensus state: current_term: 1 leader_uuid: "cb05a59f29a6477394f9db9f0aaa1699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.205824  9156 sys_catalog.cc:458] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [sys.catalog]: This master's current role is: FOLLOWER
09:03:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.223126  9141 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cb05a59f29a6477394f9db9f0aaa1699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.223919  9141 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: This master's current role is: LEADER
09:03:32.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.179749  8978 master_runner.cc:187] Error getting master registration for 127.8.175.254:35127: OK, instance_id { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" instance_seqno: 1794992607517025 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:32.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.237927  8978 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:32.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.254948  8976 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } attempt: 7
09:03:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.291520  8976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: Bootstrap starting.
09:03:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:32.303248  9005 tablet.cc:2441] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:32.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.313836  8976 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: Neither blocks nor log segments found. Creating new log.
09:03:32.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.320773  8976 log.cc:826] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: Log is configured to *not* fsync() on all Append() calls
09:03:32.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.345309  8976 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: No bootstrap required, opened a new log
09:03:32.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.422760  8976 raft_consensus.cc:360] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:32.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.425148  8976 raft_consensus.cc:386] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.426105  8976 raft_consensus.cc:748] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12e9ccf2635a49f3bb38115f14251af9, State: Initialized, Role: FOLLOWER
09:03:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.428882  8976 consensus_queue.cc:261] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } }
09:03:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.447288  9177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.448153  9177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [sys.catalog]: This master's current role is: FOLLOWER
09:03:32.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.455017  8949 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [term 0 FOLLOWER]: Advancing to term 1
09:03:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.458374  8951 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cb05a59f29a6477394f9db9f0aaa1699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "12e9ccf2635a49f3bb38115f14251af9" is_pre_election: true
09:03:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.460421  8950 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cb05a59f29a6477394f9db9f0aaa1699" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "12e9ccf2635a49f3bb38115f14251af9"
09:03:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.468839  8976 sys_catalog.cc:565] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [sys.catalog]: configured and running, proceeding with master startup.
09:03:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.469910  9171 consensus_queue.cc:1049] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [LEADER]: Connected to new peer: Peer: permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:03:32.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:32.470980  8930 tablet.cc:2441] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:32.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.507938  9177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cb05a59f29a6477394f9db9f0aaa1699. Latest consensus state: current_term: 1 leader_uuid: "cb05a59f29a6477394f9db9f0aaa1699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.508821  9177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [sys.catalog]: This master's current role is: FOLLOWER
09:03:32.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.515728  9177 sys_catalog.cc:455] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cb05a59f29a6477394f9db9f0aaa1699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.516414  9177 sys_catalog.cc:458] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9 [sys.catalog]: This master's current role is: FOLLOWER
09:03:32.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.528330  9141 sys_catalog.cc:455] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "cb05a59f29a6477394f9db9f0aaa1699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.529320  9141 sys_catalog.cc:458] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699 [sys.catalog]: This master's current role is: LEADER
09:03:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.534487  9170 catalog_manager.cc:1398] Generated new cluster ID: dd3e0d43edb248848b7ef4b1fea8b839
09:03:32.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.534801  9170 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
09:03:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.571466  9170 catalog_manager.cc:1421] Generated new certificate authority record
09:03:32.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.573988  9170 catalog_manager.cc:1555] Loading token signing keys...
09:03:32.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.606806  9193 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:03:32.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.631439  9170 catalog_manager.cc:6221] T 00000000000000000000000000000000 P cb05a59f29a6477394f9db9f0aaa1699: Generated new TSK 0
09:03:32.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.633049  9170 catalog_manager.cc:1565] Initializing in-progress tserver states...
09:03:32.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:32.826574  9027 thread.cc:642] prepare [worker] (thread pool) Time spent creating pthread: real 0.621s	user 0.005s	sys 0.003s
09:03:32.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:32.826936  9027 thread.cc:609] prepare [worker] (thread pool) Time spent starting thread: real 0.621s	user 0.005s	sys 0.003s
09:03:32.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.849438  9194 sys_catalog.cc:455] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "cb05a59f29a6477394f9db9f0aaa1699" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "12e9ccf2635a49f3bb38115f14251af9" member_type: VOTER last_known_addr { host: "127.8.175.254" port: 35127 } } peers { permanent_uuid: "db2c29d9dbf6476785737f01269e1b8a" member_type: VOTER last_known_addr { host: "127.8.175.253" port: 42151 } } peers { permanent_uuid: "cb05a59f29a6477394f9db9f0aaa1699" member_type: VOTER last_known_addr { host: "127.8.175.252" port: 45213 } } }
09:03:32.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.850279  9194 sys_catalog.cc:458] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a [sys.catalog]: This master's current role is: FOLLOWER
09:03:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:32.880712  8895 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/wal
09:03:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/logs
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data/info.pb
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.193:0
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.175.193
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.193
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:03:33.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.183619  9168 catalog_manager.cc:1310] Loaded cluster ID: dd3e0d43edb248848b7ef4b1fea8b839
09:03:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.184769  9168 catalog_manager.cc:1603] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: loading cluster ID for follower catalog manager: success
09:03:33.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.192852  9168 catalog_manager.cc:1625] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: acquiring CA information for follower catalog manager: success
09:03:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.199656  9168 catalog_manager.cc:1653] T 00000000000000000000000000000000 P db2c29d9dbf6476785737f01269e1b8a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:33.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:33.442020  9197 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:33.442775  9197 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:33.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:33.443531  9197 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:33.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:33.505987  9197 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:33.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:33.507655  9197 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.175.193
09:03:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.576826  9197 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/logs
09:03:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/wal
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.193:0
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data/info.pb
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.193
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.175.193
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.586982  9197 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:33.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.589747  9197 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:33.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:33.607017  9204 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.659912  9193 catalog_manager.cc:1310] Loaded cluster ID: dd3e0d43edb248848b7ef4b1fea8b839
09:03:33.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.660300  9193 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: loading cluster ID for follower catalog manager: success
09:03:33.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.698107  9193 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: acquiring CA information for follower catalog manager: success
09:03:33.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:33.731536  9193 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 12e9ccf2635a49f3bb38115f14251af9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:35.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:35.009004  9203 debug-util.cc:398] Leaking SignalData structure 0x7b0800029220 after lost signal to thread 9197
09:03:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:35.925282  9203 kernel_stack_watchdog.cc:198] Thread 9197 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 392ms:
09:03:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:35.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:33.617064  9205 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:35.931953  9197 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.317s	user 0.000s	sys 0.007s
09:03:35.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:35.932307  9197 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.318s	user 0.000s	sys 0.007s
09:03:35.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:35.933122  9206 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2309 milliseconds
09:03:35.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:35.934211  9209 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:35.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.934228  9197 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.935761  9197 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:35.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.938724  9197 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:35.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.940410  9197 hybrid_clock.cc:648] HybridClock initialized: now 1794992615940356 us; error 49 us; skew 500 ppm
09:03:35.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.941356  9197 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:35.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.949131  9197 webserver.cc:577] Webserver started at http://127.8.175.193:35673/ using document root <none> and password file <none>
09:03:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.950282  9197 fs_manager.cc:362] Metadata directory not provided
09:03:35.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.950539  9197 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:35.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.951049  9197 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.955590  9197 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data/instance:
09:03:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab04f8fa04c44999a4605b5397ac0b31"
09:03:35.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:35 on dist-test-slave-mh3s"
09:03:35.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.956875  9197 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/wal/instance:
09:03:35.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab04f8fa04c44999a4605b5397ac0b31"
09:03:35.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:35 on dist-test-slave-mh3s"
09:03:35.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.965903  9197 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.007s	sys 0.004s
09:03:35.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.972611  9217 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.974035  9197 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.006s	sys 0.000s
09:03:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.974462  9197 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/wal
09:03:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ab04f8fa04c44999a4605b5397ac0b31"
09:03:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:35 on dist-test-slave-mh3s"
09:03:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:35.974884  9197 fs_report.cc:389] FS layout report
09:03:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/wal
09:03:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/wal
09:03:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data/data
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:35.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:35.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:36.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.034341  9197 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:36.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.036404  9197 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:36.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.037117  9197 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:36.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.041077  9197 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:36.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.046607  9197 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.046859  9197 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
09:03:36.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.047158  9197 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.047410  9197 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
09:03:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.273676  9197 rpc_server.cc:307] RPC server started. Bound to: 127.8.175.193:43723
09:03:36.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.273787  9329 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.175.193:43723 every 8 connection(s)
09:03:36.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.277460  9197 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/data/info.pb
09:03:36.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.286779  8895 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9197
09:03:36.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.287374  8895 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-0/wal/instance
09:03:36.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.300441  8895 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/wal
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/logs
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data/info.pb
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:36.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.194:0
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.175.194
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.194
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:36.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:03:36.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.361159  9330 heartbeater.cc:345] Connected to a master server at 127.8.175.253:42151
09:03:36.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.361743  9330 heartbeater.cc:462] Registering TS with master...
09:03:36.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.363209  9330 heartbeater.cc:508] Master 127.8.175.253:42151 requested a full tablet report, sending...
09:03:36.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.369082  9016 ts_manager.cc:210] Registered new tserver with Master: ab04f8fa04c44999a4605b5397ac0b31 (127.8.175.193:43723)
09:03:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.386821  9331 heartbeater.cc:345] Connected to a master server at 127.8.175.252:45213
09:03:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.387197  9331 heartbeater.cc:462] Registering TS with master...
09:03:36.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.388098  9331 heartbeater.cc:508] Master 127.8.175.252:45213 requested a full tablet report, sending...
09:03:36.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.391512  9097 ts_manager.cc:210] Registered new tserver with Master: ab04f8fa04c44999a4605b5397ac0b31 (127.8.175.193:43723)
09:03:36.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.395258  9097 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.8.175.193:37803
09:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.415815  9335 heartbeater.cc:345] Connected to a master server at 127.8.175.254:35127
09:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.416208  9335 heartbeater.cc:462] Registering TS with master...
09:03:36.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.416931  9335 heartbeater.cc:508] Master 127.8.175.254:35127 requested a full tablet report, sending...
09:03:36.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.421546  8941 ts_manager.cc:210] Registered new tserver with Master: ab04f8fa04c44999a4605b5397ac0b31 (127.8.175.193:43723)
09:03:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.789233  9334 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:36.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.789820  9334 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:36.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.790524  9334 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.836573  9334 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:36.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.838487  9334 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.175.194
09:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.884593  9334 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/logs
09:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/wal
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.194:0
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data/info.pb
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.194
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.175.194
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:36.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.886503  9334 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:36.888847  9334 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:36.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.905473  9347 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:37.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:37.401191  9331 heartbeater.cc:500] Master 127.8.175.252:45213 was elected leader, sending a full tablet report...
09:03:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.911689  9350 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:38.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:36.908286  9348 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:38.274871  9349 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1365 milliseconds
09:03:38.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.275033  9334 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.276576  9334 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.279383  9334 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.281028  9334 hybrid_clock.cc:648] HybridClock initialized: now 1794992618280994 us; error 41 us; skew 500 ppm
09:03:38.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.282137  9334 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:38.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.296912  9334 webserver.cc:577] Webserver started at http://127.8.175.194:35807/ using document root <none> and password file <none>
09:03:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.297850  9334 fs_manager.cc:362] Metadata directory not provided
09:03:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.298038  9334 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.298518  9334 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:38.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.302996  9334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data/instance:
09:03:38.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77c2643027a9469dbfcb25a5c8a875b5"
09:03:38.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:38 on dist-test-slave-mh3s"
09:03:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.304068  9334 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/wal/instance:
09:03:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77c2643027a9469dbfcb25a5c8a875b5"
09:03:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:38 on dist-test-slave-mh3s"
09:03:38.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.316864  9334 fs_manager.cc:696] Time spent creating directory manager: real 0.012s	user 0.009s	sys 0.000s
09:03:38.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.326319  9358 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.327898  9334 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.003s	sys 0.001s
09:03:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.328305  9334 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/wal
09:03:38.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77c2643027a9469dbfcb25a5c8a875b5"
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:38 on dist-test-slave-mh3s"
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.328770  9334 fs_report.cc:389] FS layout report
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/wal
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/wal
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data/data
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:38.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.396128  9334 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.397706  9334 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.398273  9334 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:38.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.400940  9334 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.405180  9334 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.405422  9334 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
09:03:38.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.405766  9334 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.405990  9334 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
09:03:38.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.563391  9334 rpc_server.cc:307] RPC server started. Bound to: 127.8.175.194:36223
09:03:38.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.563489  9470 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.175.194:36223 every 8 connection(s)
09:03:38.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.566220  9334 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/data/info.pb
09:03:38.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.576907  8895 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9334
09:03:38.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.577456  8895 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-1/wal/instance
09:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.603808  8895 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/wal
09:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data
09:03:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/logs
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data/info.pb
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.195:0
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.175.195
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.195
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:38.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
09:03:38.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.611233  9472 heartbeater.cc:345] Connected to a master server at 127.8.175.252:45213
09:03:38.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.611743  9472 heartbeater.cc:462] Registering TS with master...
09:03:38.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.613154  9472 heartbeater.cc:508] Master 127.8.175.252:45213 requested a full tablet report, sending...
09:03:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.615479  9471 heartbeater.cc:345] Connected to a master server at 127.8.175.253:42151
09:03:38.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.615820  9471 heartbeater.cc:462] Registering TS with master...
09:03:38.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.616704  9471 heartbeater.cc:508] Master 127.8.175.253:42151 requested a full tablet report, sending...
09:03:38.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.617702  9097 ts_manager.cc:210] Registered new tserver with Master: 77c2643027a9469dbfcb25a5c8a875b5 (127.8.175.194:36223)
09:03:38.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.618242  9473 heartbeater.cc:345] Connected to a master server at 127.8.175.254:35127
09:03:38.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.618551  9473 heartbeater.cc:462] Registering TS with master...
09:03:38.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.619385  9473 heartbeater.cc:508] Master 127.8.175.254:35127 requested a full tablet report, sending...
09:03:38.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.619472  9097 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.8.175.194:40419
09:03:38.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.619740  9016 ts_manager.cc:210] Registered new tserver with Master: 77c2643027a9469dbfcb25a5c8a875b5 (127.8.175.194:36223)
09:03:38.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:38.622733  8941 ts_manager.cc:210] Registered new tserver with Master: 77c2643027a9469dbfcb25a5c8a875b5 (127.8.175.194:36223)
09:03:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:38.942350  9481 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:38.942942  9481 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:38.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:38.943403  9481 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:38.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:38.977882  9481 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:38.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:38.978808  9481 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.8.175.195
09:03:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:39.017299  9481 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/logs
09:03:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.8.175.212:36143
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/wal
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.8.175.195:0
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:39.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data/info.pb
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.8.175.195
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.8.175.254:35127,127.8.175.253:42151,127.8.175.252:45213
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.8.175.195
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:39.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:39.018877  9481 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:39.020536  9481 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:39.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:39.035594  9487 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:39.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:39.625016  9472 heartbeater.cc:500] Master 127.8.175.252:45213 was elected leader, sending a full tablet report...
09:03:40.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:40.443253  9486 debug-util.cc:398] Leaking SignalData structure 0x7b0800029220 after lost signal to thread 9481
09:03:40.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:39.036548  9488 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:40.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:40.545173  9481 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.510s	user 0.537s	sys 0.947s
09:03:40.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:40.547030  9481 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.512s	user 0.537s	sys 0.947s
09:03:40.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:40.547194  9489 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1510 milliseconds
09:03:40.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.547356  9481 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:40.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:40.547458  9490 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:40.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.552172  9481 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:40.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.554431  9481 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:40.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.555815  9481 hybrid_clock.cc:648] HybridClock initialized: now 1794992620555772 us; error 50 us; skew 500 ppm
09:03:40.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.556663  9481 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.564371  9481 webserver.cc:577] Webserver started at http://127.8.175.195:37373/ using document root <none> and password file <none>
09:03:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.565639  9481 fs_manager.cc:362] Metadata directory not provided
09:03:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.565915  9481 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.566497  9481 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:40.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.572950  9481 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data/instance:
09:03:40.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1dab1f0edf414ec78acd9cd30ed14f02"
09:03:40.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:40 on dist-test-slave-mh3s"
09:03:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.574702  9481 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/wal/instance:
09:03:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1dab1f0edf414ec78acd9cd30ed14f02"
09:03:40.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:40 on dist-test-slave-mh3s"
09:03:40.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.583532  9481 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.000s	sys 0.007s
09:03:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.589407  9498 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.590651  9481 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.002s	sys 0.001s
09:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.590909  9481 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/wal
09:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1dab1f0edf414ec78acd9cd30ed14f02"
09:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:40 on dist-test-slave-mh3s"
09:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.591207  9481 fs_report.cc:389] FS layout report
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/wal
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/wal
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data/data
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.648734  9481 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.650291  9481 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.650717  9481 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.653437  9481 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.657609  9481 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.657810  9481 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
09:03:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.658017  9481 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.658195  9481 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.001s	sys 0.000s
09:03:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.915946  9481 rpc_server.cc:307] RPC server started. Bound to: 127.8.175.195:37337
09:03:40.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.916141  9610 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.8.175.195:37337 every 8 connection(s)
09:03:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.919394  9481 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/data/info.pb
09:03:40.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.924674  8895 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9481
09:03:40.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.925200  8895 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster13054957754238416071/ts-2/wal/instance
09:03:40.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.999260  9615 heartbeater.cc:345] Connected to a master server at 127.8.175.254:35127
09:03:41.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:40.999995  9615 heartbeater.cc:462] Registering TS with master...
09:03:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.000999  9612 heartbeater.cc:345] Connected to a master server at 127.8.175.252:45213
09:03:41.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.001490  9612 heartbeater.cc:462] Registering TS with master...
09:03:41.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.003248  9612 heartbeater.cc:508] Master 127.8.175.252:45213 requested a full tablet report, sending...
09:03:41.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.003402  9615 heartbeater.cc:508] Master 127.8.175.254:35127 requested a full tablet report, sending...
09:03:41.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.006202  9611 heartbeater.cc:345] Connected to a master server at 127.8.175.253:42151
09:03:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.006748  9611 heartbeater.cc:462] Registering TS with master...
09:03:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.007988  9097 ts_manager.cc:210] Registered new tserver with Master: 1dab1f0edf414ec78acd9cd30ed14f02 (127.8.175.195:37337)
09:03:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.008477  9611 heartbeater.cc:508] Master 127.8.175.253:42151 requested a full tablet report, sending...
09:03:41.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.009501  8941 ts_manager.cc:210] Registered new tserver with Master: 1dab1f0edf414ec78acd9cd30ed14f02 (127.8.175.195:37337)
09:03:41.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.010056  9097 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.8.175.195:46553
09:03:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.012410  9016 ts_manager.cc:210] Registered new tserver with Master: 1dab1f0edf414ec78acd9cd30ed14f02 (127.8.175.195:37337)
09:03:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.028978  8895 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
09:03:41.049 [DEBUG - main] (MiniKuduCluster.java:195) Response: 
09:03:41.050 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}

09:03:41.061 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.8.175.254"
      port: 35127
    }
    bound_http_address {
      host: "127.8.175.254"
      port: 38889
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.8.175.253"
      port: 42151
    }
    bound_http_address {
      host: "127.8.175.253"
      port: 40411
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.8.175.252"
      port: 45213
    }
    bound_http_address {
      host: "127.8.175.252"
      port: 34341
    }
  }
}

09:03:41.065 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}

09:03:41.068 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.8.175.193"
      port: 43723
    }
    bound_http_address {
      host: "127.8.175.193"
      port: 35673
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.8.175.194"
      port: 36223
    }
    bound_http_address {
      host: "127.8.175.194"
      port: 35807
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.8.175.195"
      port: 37337
    }
    bound_http_address {
      host: "127.8.175.195"
      port: 37373
    }
  }
}

09:03:41.068 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
09:03:41.243 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
	at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
	at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
	at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
	at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
	at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
	at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
	at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
	at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
	at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
	at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
	at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runners.Suite.runChild(Suite.java:128)
	at org.junit.runners.Suite.runChild(Suite.java:27)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
	at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
	at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
	at org.junit.runner.JUnitCore.main(JUnitCore.java:36)

09:03:41.309 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication 
09:03:41.403 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskhl46EI/test-tmp
09:03:41.404 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
09:03:41.404 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskhl46EI/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
09:03:41.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
  num_masters: 3
  num_tservers: 3
  enable_kerberos: true
  cluster_root: "/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553"
  extra_master_flags: "--webserver_require_spnego=false"
  extra_master_flags: "--authn_token_validity_seconds=10"
  extra_tserver_flags: "--webserver_require_spnego=false"
  hms_mode: NONE
  mini_kdc_options {
    ticket_lifetime: "10s"
    renew_lifetime: "20s"
  }
  principal: "kudu"
  mini_oidc_options {
  }
}

09:03:41.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:03:41.902 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}

09:03:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.907775  9623 mini_oidc.cc:150] Starting JWKS server
09:03:41.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.912235  9623 webserver.cc:577] Webserver started at http://127.0.0.1:39511/ using document root <none> and password file <none>
09:03:41.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.912809  9623 mini_oidc.cc:186] Starting OIDC Discovery server
09:03:41.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:41.915146  9623 webserver.cc:577] Webserver started at http://127.0.0.1:45433/ using document root <none> and password file <none>
09:03:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:42.015645  9612 heartbeater.cc:500] Master 127.8.175.252:45213 was elected leader, sending a full tablet report...
09:03:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
09:03:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/principal' for realm 'KRBTEST.COM',
09:03:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
09:03:42.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:42 dist-test-slave-mh3s krb5kdc[9635](info): setting up network...
09:03:42.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
09:03:42.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:42 dist-test-slave-mh3s krb5kdc[9635](info): set up 2 sockets
09:03:42.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:42 dist-test-slave-mh3s krb5kdc[9635](info): commencing operation
09:03:42.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
09:03:44.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:44.336181  9623 mini_kdc.cc:121] Time spent starting KDC: real 2.381s	user 0.003s	sys 0.016s
09:03:44.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
09:03:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
09:03:44.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
09:03:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:44.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
09:03:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
09:03:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
09:03:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
09:03:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/test-user.keytab.
09:03:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/test-user.keytab.
09:03:44.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:44 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992624, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:44.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
09:03:44.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T09:03:44Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
09:03:44.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T09:03:44Z Disabled control of system clock
09:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.101.254@KRBTEST.COM; defaulting to no policy
09:03:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:44.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.101.254@KRBTEST.COM" created.
09:03:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:44.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.101.254@KRBTEST.COM; defaulting to no policy
09:03:44.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:44.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.101.254@KRBTEST.COM" created.
09:03:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:44.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:44.578944  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/logs
09:03:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/info.pb
09:03:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:44.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.254:43661
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.254
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.254
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:44.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:44.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:44.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:03:44.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:44.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.000588  9652 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.001475  9652 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.002193  9652 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.039275  9652 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.039518  9652 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.039819  9652 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.040023  9652 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.040231  9652 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.040445  9652 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:45.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.040599  9652 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39511/jwks.json
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:45.080842  9652 master_runner.cc:387] Master server non-default flags:
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/logs
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.254
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.254:43661
09:03:45.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/info.pb
09:03:45.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.254
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:45.082228  9652 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:45.083220  9652 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:45.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.094962  9659 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:46.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:45.096010  9660 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:46.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:46.406054  9662 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:46.409255  9661 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1308 milliseconds
09:03:46.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.409377  9652 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:46.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.410792  9652 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:46.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.413887  9652 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:46.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.415442  9652 hybrid_clock.cc:648] HybridClock initialized: now 1794992626415386 us; error 60 us; skew 500 ppm
09:03:46.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.416743  9652 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:46.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:46 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992626, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:46.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.423661  9652 init.cc:377] Logged in from keytab as kudu/127.9.101.254@KRBTEST.COM (short username kudu)
09:03:46.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.428997  9652 webserver.cc:577] Webserver started at http://127.9.101.254:34349/ using document root <none> and password file <none>
09:03:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.429976  9652 fs_manager.cc:362] Metadata directory not provided
09:03:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.430249  9652 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.430797  9652 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.435078  9652 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/instance:
09:03:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d040b101867c474ea9335529606211ff"
09:03:46.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:46 on dist-test-slave-mh3s"
09:03:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.436249  9652 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal/instance:
09:03:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d040b101867c474ea9335529606211ff"
09:03:46.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:46 on dist-test-slave-mh3s"
09:03:46.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.444072  9652 fs_manager.cc:696] Time spent creating directory manager: real 0.007s	user 0.004s	sys 0.004s
09:03:46.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.449723  9671 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:46.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.451594  9652 fs_manager.cc:730] Time spent opening block manager: real 0.004s	user 0.003s	sys 0.001s
09:03:46.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.451995  9652 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:03:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d040b101867c474ea9335529606211ff"
09:03:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:46 on dist-test-slave-mh3s"
09:03:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.453085  9652 fs_report.cc:389] FS layout report
09:03:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/data
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:46.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.506004  9652 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:46.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.515605  9652 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:46.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.516101  9652 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:46.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.726361  9652 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.254:43661
09:03:46.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.726549  9722 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.254:43661 every 8 connection(s)
09:03:46.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.729372  9652 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/info.pb
09:03:46.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.737883  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9652
09:03:46.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.738556  9623 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal/instance
09:03:46.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.740681  9723 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.101.253@KRBTEST.COM; defaulting to no policy
09:03:46.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:46.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.101.253@KRBTEST.COM" created.
09:03:46.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.765754  9723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } has no permanent_uuid. Determining permanent_uuid...
09:03:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:46 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992626, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:03:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:46.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.811795  9723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } has no permanent_uuid. Determining permanent_uuid...
09:03:46.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.101.253@KRBTEST.COM; defaulting to no policy
09:03:46.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:46.814849  9675 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:46.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:46.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.101.253@KRBTEST.COM" created.
09:03:46.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:46.820689  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:46.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:46.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.845791  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:46.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/logs
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/info.pb
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:46.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.253:38469
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.253
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.253
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:46.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:03:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:46.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.870379  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 1
09:03:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:46.874647  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:46.943066  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 2
09:03:46.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:46.947868  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:47.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:47.039368  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 3
09:03:47.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.049327  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:47.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:47.192905  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 4
09:03:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.199024  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:47.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.272080  9733 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:47.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.272857  9733 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:47.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.273638  9733 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:47.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.308706  9733 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:47.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.309048  9733 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:47.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.309573  9733 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.309921  9733 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.310297  9733 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.310616  9733 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:47.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.310894  9733 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39511/jwks.json
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:47.354902  9733 master_runner.cc:387] Master server non-default flags:
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/logs
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:47.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.253
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.253:38469
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/info.pb
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.253
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:47.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:47.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:47.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:47.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:47.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:47.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:47.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:47.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:47.356717  9733 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:47.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:47.357923  9733 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:47.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.375326  9739 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:47.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:47.498474  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 5
09:03:47.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.509774  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:48.057437  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 6
09:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:48.076122  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:49.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.136715  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 7
09:03:49.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.165814  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:03:49.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:48.780772  9738 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 9733
09:03:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.295433  9733 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.915s	user 0.642s	sys 1.200s
09:03:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.296831  9733 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.916s	user 0.642s	sys 1.201s
09:03:49.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:47.380760  9740 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.298609  9741 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1915 milliseconds
09:03:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.299736  9733 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.299790  9742 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:49.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.302876  9733 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:49.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.306012  9733 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:49.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.307451  9733 hybrid_clock.cc:648] HybridClock initialized: now 1794992629307420 us; error 45 us; skew 500 ppm
09:03:49.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.308466  9733 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:49.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:49 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992629, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.316840  9733 init.cc:377] Logged in from keytab as kudu/127.9.101.253@KRBTEST.COM (short username kudu)
09:03:49.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.325853  9733 webserver.cc:577] Webserver started at http://127.9.101.253:33633/ using document root <none> and password file <none>
09:03:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.327097  9733 fs_manager.cc:362] Metadata directory not provided
09:03:49.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.327378  9733 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:49.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.327929  9733 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.333416  9733 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/instance:
09:03:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b39e81019bee4f5a986f915847561ded"
09:03:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:49 on dist-test-slave-mh3s"
09:03:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.334707  9733 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal/instance:
09:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b39e81019bee4f5a986f915847561ded"
09:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:49 on dist-test-slave-mh3s"
09:03:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.345577  9733 fs_manager.cc:696] Time spent creating directory manager: real 0.010s	user 0.001s	sys 0.009s
09:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.353969  9757 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:49.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.355315  9733 fs_manager.cc:730] Time spent opening block manager: real 0.006s	user 0.003s	sys 0.004s
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.355763  9733 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b39e81019bee4f5a986f915847561ded"
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:49 on dist-test-slave-mh3s"
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.356137  9733 fs_report.cc:389] FS layout report
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/data
09:03:49.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:49.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:49.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.433351  9733 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:49.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.440671  9733 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:49.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.441362  9733 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:49.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.652601  9733 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.253:38469
09:03:49.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.652732  9808 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.253:38469 every 8 connection(s)
09:03:49.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.657147  9733 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/info.pb
09:03:49.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.666842  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9733
09:03:49.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.667495  9623 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal/instance
09:03:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.671506  9809 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:49.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.101.252@KRBTEST.COM; defaulting to no policy
09:03:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.707257  9809 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } has no permanent_uuid. Determining permanent_uuid...
09:03:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.101.252@KRBTEST.COM" created.
09:03:49.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:49.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:49.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:49.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:49 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992629, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:03:49.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.101.252@KRBTEST.COM; defaulting to no policy
09:03:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.101.252@KRBTEST.COM" created.
09:03:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.766264  9809 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } has no permanent_uuid. Determining permanent_uuid...
09:03:49.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:49 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992629, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:03:49.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:49.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:49.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.794490  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/logs
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/info.pb
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:03:49.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.252:44553
09:03:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.252
09:03:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.252
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:03:49.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:49.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.805261  9809 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } has no permanent_uuid. Determining permanent_uuid...
09:03:49.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.809211  9758 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:49.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.819115  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:49.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.868777  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 1
09:03:49.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.875554  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:49.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:49.944084  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 2
09:03:49.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.944676  9607 debug-util.cc:398] Leaking SignalData structure 0x7b08000adde0 after lost signal to thread 9482
09:03:49.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.946440  9607 debug-util.cc:398] Leaking SignalData structure 0x7b08000b58e0 after lost signal to thread 9610
09:03:49.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:49.952901  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:50.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:50.044541  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 3
09:03:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.054450  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:50.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:50.198108  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 4
09:03:50.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.206212  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:50.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.285945  9818 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.286934  9818 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:50.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.287608  9818 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.326453  9818 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:03:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.326736  9818 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:03:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.327122  9818 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.327411  9818 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:03:50.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.327639  9818 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:03:50.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.327872  9818 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:03:50.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.328126  9818 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39511/jwks.json
09:03:50.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:50.371461  9818 master_runner.cc:387] Master server non-default flags:
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/logs
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data
09:03:50.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:03:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:03:50.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.252
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.252:44553
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:50.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/info.pb
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.252
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:50.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:50.373193  9818 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:50.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:50.374423  9818 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:50.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.401646  9826 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:50.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:50.505787  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 5
09:03:50.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.522704  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:51.070376  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 6
09:03:51.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:51.138928  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:51.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:51.256392  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } attempt: 8
09:03:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:51 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992626, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:03:51.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:51.340986  9723 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } has no permanent_uuid. Determining permanent_uuid...
09:03:51.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:51.372279  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:51.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:51.437955  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 1
09:03:51.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:51.457209  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:51.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:51.510891  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 2
09:03:51.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:51.523506  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:51.600203  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 3
09:03:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:51.612762  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:51.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:51.768285  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 4
09:03:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:51.780753  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:52.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.077311  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 5
09:03:52.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.082459  9672 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111) [suppressed 12 similar messages]
09:03:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.089720  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.199611  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 7
09:03:52.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.246300  9809 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:52.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.611383  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 6
09:03:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.647555  9723 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:03:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:51.794701  9825 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 9818
09:03:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.676376  9825 kernel_stack_watchdog.cc:198] Thread 9818 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 387ms:
09:03:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:52.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:52.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.678052  9818 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.274s	user 0.677s	sys 1.147s
09:03:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.679163  9818 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.275s	user 0.677s	sys 1.148s
09:03:52.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:50.404855  9828 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:52.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.681367  9830 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:52.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:52.681214  9829 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2271 milliseconds
09:03:52.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.683599  9818 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:03:52.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.687189  9818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:52.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.691450  9818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.692916  9818 hybrid_clock.cc:648] HybridClock initialized: now 1794992632692768 us; error 156 us; skew 500 ppm
09:03:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.694177  9818 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:52 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992632, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:52.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.703649  9818 init.cc:377] Logged in from keytab as kudu/127.9.101.252@KRBTEST.COM (short username kudu)
09:03:52.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.736687  9818 webserver.cc:577] Webserver started at http://127.9.101.252:41731/ using document root <none> and password file <none>
09:03:52.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.738816  9818 fs_manager.cc:362] Metadata directory not provided
09:03:52.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.739112  9818 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:52.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.739799  9818 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:52.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.748863  9818 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/instance:
09:03:52.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80cfa9011505477a9da4db0c6f01017d"
09:03:52.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:52 on dist-test-slave-mh3s"
09:03:52.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.750931  9818 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal/instance:
09:03:52.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80cfa9011505477a9da4db0c6f01017d"
09:03:52.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:52 on dist-test-slave-mh3s"
09:03:52.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.767082  9818 fs_manager.cc:696] Time spent creating directory manager: real 0.015s	user 0.005s	sys 0.009s
09:03:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.778391  9848 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.780043  9818 fs_manager.cc:730] Time spent opening block manager: real 0.008s	user 0.008s	sys 0.000s
09:03:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.780531  9818 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:03:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80cfa9011505477a9da4db0c6f01017d"
09:03:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:52 on dist-test-slave-mh3s"
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.781111  9818 fs_report.cc:389] FS layout report
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/data
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:52.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.900691  9818 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.918648  9818 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:52.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:52.919436  9818 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:53.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.183311  9818 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.252:44553
09:03:53.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.183523  9899 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.252:44553 every 8 connection(s)
09:03:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.188539  9818 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/info.pb
09:03:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.194743  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9818
09:03:53.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.196060  9623 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal/instance
09:03:53.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.207223  9900 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
09:03:53.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.242023  9900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } has no permanent_uuid. Determining permanent_uuid...
09:03:53.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:53 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992624, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:03:53.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:53 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992632, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.252@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:03:53.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.314620  9900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } has no permanent_uuid. Determining permanent_uuid...
09:03:53.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:53 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992624, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:03:53.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:53 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992632, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.252@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:03:53.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:53 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992624, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:03:53.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.358930  9900 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } has no permanent_uuid. Determining permanent_uuid...
09:03:53.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:53 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992632, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.252@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:03:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.420219  9900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Bootstrap starting.
09:03:53.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.436304  9900 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Neither blocks nor log segments found. Creating new log.
09:03:53.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.439000  9900 log.cc:826] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Log is configured to *not* fsync() on all Append() calls
09:03:53.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.449965  9900 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: No bootstrap required, opened a new log
09:03:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.487596  9900 raft_consensus.cc:360] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:53.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.488524  9900 raft_consensus.cc:386] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:53.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.488912  9900 raft_consensus.cc:748] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80cfa9011505477a9da4db0c6f01017d, State: Initialized, Role: FOLLOWER
09:03:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.490069  9900 consensus_queue.cc:261] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.495728  9912 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.497118  9912 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: This master's current role is: FOLLOWER
09:03:53.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.499421  9900 sys_catalog.cc:565] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: configured and running, proceeding with master startup.
09:03:53.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:53.585490  9924 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:53.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:53.586303  9924 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.558736  9818 master_runner.cc:187] Error getting master registration for 127.9.101.254:43661: OK, instance_id { permanent_uuid: "d040b101867c474ea9335529606211ff" instance_seqno: 1794992626671615 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:53.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.605396  9818 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:53.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:53.685140  9854 tablet.cc:2441] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:53.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.685139  9723 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 7
09:03:53.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:53 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992626, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:03:53.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.818159  9723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Bootstrap starting.
09:03:53.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.834686  9723 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Neither blocks nor log segments found. Creating new log.
09:03:53.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.840138  9723 log.cc:826] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Log is configured to *not* fsync() on all Append() calls
09:03:53.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.848569  9723 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: No bootstrap required, opened a new log
09:03:53.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.897656  9723 raft_consensus.cc:360] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:53.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.898864  9723 raft_consensus.cc:386] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:53.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.899318  9723 raft_consensus.cc:748] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d040b101867c474ea9335529606211ff, State: Initialized, Role: FOLLOWER
09:03:53.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.900748  9723 consensus_queue.cc:261] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.903710  9928 sys_catalog.cc:455] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.904436  9928 sys_catalog.cc:458] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: This master's current role is: FOLLOWER
09:03:53.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.907191  9723 sys_catalog.cc:565] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: configured and running, proceeding with master startup.
09:03:53.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.930930  9912 raft_consensus.cc:499] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:03:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.932044  9912 raft_consensus.cc:521] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.938576  9912 leader_election.cc:290] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d040b101867c474ea9335529606211ff (127.9.101.254:43661), b39e81019bee4f5a986f915847561ded (127.9.101.253:38469)
09:03:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.939327  9698 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80cfa9011505477a9da4db0c6f01017d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d040b101867c474ea9335529606211ff" is_pre_election: true
09:03:53.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.940757  9698 raft_consensus.cc:2490] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 80cfa9011505477a9da4db0c6f01017d in term 0.
09:03:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.943794  9850 leader_election.cc:304] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 80cfa9011505477a9da4db0c6f01017d, d040b101867c474ea9335529606211ff; no voters: 
09:03:53.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.945969  9912 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Leader pre-election won for term 1
09:03:53.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.947264  9912 raft_consensus.cc:499] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:03:53.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.948369  9912 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 0 FOLLOWER]: Advancing to term 1
09:03:53.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.966773  9912 raft_consensus.cc:521] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:53.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.983520  9698 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80cfa9011505477a9da4db0c6f01017d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d040b101867c474ea9335529606211ff"
09:03:53.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.984294  9698 raft_consensus.cc:3096] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 0 FOLLOWER]: Advancing to term 1
09:03:53.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:53.984438  9912 leader_election.cc:290] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [CANDIDATE]: Term 1 election: Requested vote from peers d040b101867c474ea9335529606211ff (127.9.101.254:43661), b39e81019bee4f5a986f915847561ded (127.9.101.253:38469)
09:03:54.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.001999  9698 raft_consensus.cc:2490] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 80cfa9011505477a9da4db0c6f01017d in term 1.
09:03:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.004027  9850 leader_election.cc:304] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 80cfa9011505477a9da4db0c6f01017d, d040b101867c474ea9335529606211ff; no voters: 
09:03:54.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.005548  9912 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 1 FOLLOWER]: Leader election won for term 1
09:03:54.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.010263  9912 raft_consensus.cc:705] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 1 LEADER]: Becoming Leader. State: Replica: 80cfa9011505477a9da4db0c6f01017d, State: Running, Role: LEADER
09:03:54.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.012059  9912 consensus_queue.cc:238] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:54.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.035869  9677 tablet.cc:2441] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.034484  9939 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80cfa9011505477a9da4db0c6f01017d. Latest consensus state: current_term: 1 leader_uuid: "80cfa9011505477a9da4db0c6f01017d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:54.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.052570  9939 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: This master's current role is: LEADER
09:03:54.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.057904  9912 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
09:03:54.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.100248  9941 catalog_manager.cc:1609] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
09:03:54.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.100780  9941 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
09:03:54.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.101804  9945 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
09:03:54.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.088665  9928 sys_catalog.cc:455] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80cfa9011505477a9da4db0c6f01017d. Latest consensus state: current_term: 1 leader_uuid: "80cfa9011505477a9da4db0c6f01017d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:54.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.102990  9928 sys_catalog.cc:458] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: This master's current role is: FOLLOWER
09:03:54.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.053900  9652 master_runner.cc:187] Error getting master registration for 127.9.101.253:38469: OK, instance_id { permanent_uuid: "b39e81019bee4f5a986f915847561ded" instance_seqno: 1794992629592595 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
09:03:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.162357  9652 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:03:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.168918  9928 sys_catalog.cc:455] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "80cfa9011505477a9da4db0c6f01017d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.169970  9928 sys_catalog.cc:458] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: This master's current role is: FOLLOWER
09:03:54.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.180341  9945 catalog_manager.cc:1535] Initializing Kudu cluster ID...
09:03:54.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.194625  9939 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "80cfa9011505477a9da4db0c6f01017d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:54.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.195785  9939 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: This master's current role is: LEADER
09:03:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.293884  9945 catalog_manager.cc:1398] Generated new cluster ID: 34808d01116841a5a7aa8fa392bac2de
09:03:54.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.294377  9945 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
09:03:54.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.336997  9809 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } attempt: 8
09:03:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.387126  9945 catalog_manager.cc:1421] Generated new certificate authority record
09:03:54.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.400970  9945 catalog_manager.cc:1555] Loading token signing keys...
09:03:54.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:54 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992629, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:03:54.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.499207  9945 catalog_manager.cc:6221] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Generated new TSK 0
09:03:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.499393  9809 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Bootstrap starting.
09:03:54.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.501564  9945 catalog_manager.cc:1565] Initializing in-progress tserver states...
09:03:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.534282  9809 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Neither blocks nor log segments found. Creating new log.
09:03:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.541392  9809 log.cc:826] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Log is configured to *not* fsync() on all Append() calls
09:03:54.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.566602  9809 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: No bootstrap required, opened a new log
09:03:54.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.101.193@KRBTEST.COM; defaulting to no policy
09:03:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:54.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.101.193@KRBTEST.COM" created.
09:03:54.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.634578  9809 raft_consensus.cc:360] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.636058  9809 raft_consensus.cc:386] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
09:03:54.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.636617  9809 raft_consensus.cc:748] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b39e81019bee4f5a986f915847561ded, State: Initialized, Role: FOLLOWER
09:03:54.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.638202  9809 consensus_queue.cc:261] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:03:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:54.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.677793  9623 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.9.101.193: real 0.129s	user 0.000s	sys 0.025s
09:03:54.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.679158  9956 sys_catalog.cc:455] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:54.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.682494  9956 sys_catalog.cc:458] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: This master's current role is: FOLLOWER
09:03:54.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.690104  9809 sys_catalog.cc:565] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: configured and running, proceeding with master startup.
09:03:54.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.691272  9783 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80cfa9011505477a9da4db0c6f01017d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b39e81019bee4f5a986f915847561ded"
09:03:54.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.692440  9783 raft_consensus.cc:3096] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 0 FOLLOWER]: Advancing to term 1
09:03:54.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.690647  9784 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "80cfa9011505477a9da4db0c6f01017d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b39e81019bee4f5a986f915847561ded" is_pre_election: true
09:03:54.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.708334  9763 tablet.cc:2441] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Can't schedule compaction. Clean time has not been advanced past its initial value.
09:03:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.725915  9783 raft_consensus.cc:2490] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 80cfa9011505477a9da4db0c6f01017d in term 1.
09:03:54.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.101.193@KRBTEST.COM; defaulting to no policy
09:03:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:54.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.101.193@KRBTEST.COM" created.
09:03:54.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.741617  9939 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
09:03:54.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.786950  8972 debug-util.cc:398] Leaking SignalData structure 0x7b080009a300 after lost signal to thread 8908
09:03:54.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.787716  8972 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5d40 after lost signal to thread 8975
09:03:54.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:03:54.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:54.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:54.806650  9623 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.9.101.193: real 0.128s	user 0.003s	sys 0.012s
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.809903  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/wal
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/logs
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data/info.pb
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:03:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.193:0
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.101.193
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.193
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.193
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:03:54.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.923990  9956 sys_catalog.cc:455] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: SysCatalogTable state changed. Reason: New leader 80cfa9011505477a9da4db0c6f01017d. Latest consensus state: current_term: 1 leader_uuid: "80cfa9011505477a9da4db0c6f01017d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.924741  9969 sys_catalog.cc:455] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "80cfa9011505477a9da4db0c6f01017d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.926278  9969 sys_catalog.cc:458] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: This master's current role is: FOLLOWER
09:03:54.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.926275  9956 sys_catalog.cc:458] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: This master's current role is: FOLLOWER
09:03:54.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:54.945672  9975 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:03:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.108778  9941 catalog_manager.cc:1310] Loaded cluster ID: 34808d01116841a5a7aa8fa392bac2de
09:03:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.109212  9941 catalog_manager.cc:1603] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: loading cluster ID for follower catalog manager: success
09:03:55.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.117408  9941 catalog_manager.cc:1625] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: acquiring CA information for follower catalog manager: success
09:03:55.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.147217  9941 catalog_manager.cc:1653] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.315364  9939 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "80cfa9011505477a9da4db0c6f01017d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:03:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.316337  9939 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: This master's current role is: LEADER
09:03:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:55.662309  9967 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:03:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:55.663075  9967 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:03:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:55.664319  9967 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:03:55.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:55.776880  9967 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:03:55.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:55.778604  9967 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.101.193
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.913764  9967 tablet_server_runner.cc:78] Tablet server non-default flags:
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/logs
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/wal
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.193
09:03:55.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.193:0
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data/info.pb
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.193
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:03:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.101.193
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:03:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:03:55.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:03:55.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.915704  9967 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:03:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.917150  9967 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:03:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.975483  9975 catalog_manager.cc:1310] Loaded cluster ID: 34808d01116841a5a7aa8fa392bac2de
09:03:55.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:55.975848  9975 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: loading cluster ID for follower catalog manager: success
09:03:55.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:55.979578  9986 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:56.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:56.015079  9975 catalog_manager.cc:1625] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: acquiring CA information for follower catalog manager: success
09:03:56.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:56.036742  9975 catalog_manager.cc:1653] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:03:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:56 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992636, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:56.431217  9667 init.cc:295] Successfully reacquired a new kerberos TGT
09:03:58.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:57.346323  9985 debug-util.cc:398] Leaking SignalData structure 0x7b080002efa0 after lost signal to thread 9967
09:03:58.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:58.876463  9985 kernel_stack_watchdog.cc:198] Thread 9967 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 353ms:
09:03:58.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:03:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:03:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:03:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:03:58.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:55.990737  9988 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:58.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:58.880910  9967 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 2.890s	user 0.608s	sys 1.196s
09:03:58.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:58.881572  9967 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 2.891s	user 0.609s	sys 1.196s
09:03:58.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:58.894447  9967 server_base.cc:1061] running on GCE node
09:03:58.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:03:58.906821  9995 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:03:58.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:58.911842  9967 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:03:58.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:58.931742  9967 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:03:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:58.949978  9967 hybrid_clock.cc:648] HybridClock initialized: now 1794992638949887 us; error 109 us; skew 500 ppm
09:03:58.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:58.951440  9967 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:03:58.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:58 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992638, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:58.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:58.975343  9967 init.cc:377] Logged in from keytab as kudu/127.9.101.193@KRBTEST.COM (short username kudu)
09:03:59.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.013670  9967 webserver.cc:577] Webserver started at http://127.9.101.193:36043/ using document root <none> and password file <none>
09:03:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.015125  9967 fs_manager.cc:362] Metadata directory not provided
09:03:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.015439  9967 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:03:59.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.016100  9967 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:03:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.023456  9967 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data/instance:
09:03:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80ea234634674162832af7e36b98a453"
09:03:59.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:59 on dist-test-slave-mh3s"
09:03:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.025308  9967 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/wal/instance:
09:03:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80ea234634674162832af7e36b98a453"
09:03:59.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:59 on dist-test-slave-mh3s"
09:03:59.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.048679  9967 fs_manager.cc:696] Time spent creating directory manager: real 0.022s	user 0.017s	sys 0.003s
09:03:59.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.068114 10002 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:03:59.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.070039  9967 fs_manager.cc:730] Time spent opening block manager: real 0.016s	user 0.000s	sys 0.011s
09:03:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.070873  9967 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/wal
09:03:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80ea234634674162832af7e36b98a453"
09:03:59.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:59 on dist-test-slave-mh3s"
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.071462  9967 fs_report.cc:389] FS layout report
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/wal
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/wal
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data/data
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:03:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:03:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.254419  9967 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:03:59.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.262430  9967 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:03:59.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.263015  9967 kserver.cc:163] Server-wide thread pool size limit: 3276
09:03:59.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.272570  9967 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:03:59.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.284482  9967 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:03:59.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.284801  9967 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
09:03:59.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.285112  9967 ts_tablet_manager.cc:662] Registered 0 tablets
09:03:59.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.285354  9967 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
09:03:59.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:03:59 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992639, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:03:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.325490  9753 init.cc:295] Successfully reacquired a new kerberos TGT
09:03:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.945483  9967 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.193:36145
09:03:59.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.946157 10114 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.193:36145 every 8 connection(s)
09:03:59.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.949746  9967 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/data/info.pb
09:03:59.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.955147  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 9967
09:03:59.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:03:59.955657  9623 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-0/wal/instance
09:04:00.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.101.194@KRBTEST.COM; defaulting to no policy
09:04:00.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:00.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.101.194@KRBTEST.COM" created.
09:04:00.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:00.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:00.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:00 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992638, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.193@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:04:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:00 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992638, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.193@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:04:00.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:00 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992638, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.193@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:04:00.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.114954 10116 heartbeater.cc:345] Connected to a master server at 127.9.101.253:38469
09:04:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.115720 10116 heartbeater.cc:462] Registering TS with master...
09:04:00.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.117641 10115 heartbeater.cc:345] Connected to a master server at 127.9.101.252:44553
09:04:00.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.118208 10115 heartbeater.cc:462] Registering TS with master...
09:04:00.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.118779 10120 heartbeater.cc:345] Connected to a master server at 127.9.101.254:43661
09:04:00.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.119549 10120 heartbeater.cc:462] Registering TS with master...
09:04:00.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.121019 10116 heartbeater.cc:508] Master 127.9.101.253:38469 requested a full tablet report, sending...
09:04:00.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.121398 10120 heartbeater.cc:508] Master 127.9.101.254:43661 requested a full tablet report, sending...
09:04:00.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.122107 10115 heartbeater.cc:508] Master 127.9.101.252:44553 requested a full tablet report, sending...
09:04:00.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.101.194@KRBTEST.COM; defaulting to no policy
09:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.101.194@KRBTEST.COM" created.
09:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.127113  9688 ts_manager.cc:210] Registered new tserver with Master: 80ea234634674162832af7e36b98a453 (127.9.101.193:36145)
09:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.127142  9865 ts_manager.cc:210] Registered new tserver with Master: 80ea234634674162832af7e36b98a453 (127.9.101.193:36145)
09:04:00.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.127538  9774 ts_manager.cc:210] Registered new tserver with Master: 80ea234634674162832af7e36b98a453 (127.9.101.193:36145)
09:04:00.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.130592  9865 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.9.101.193@KRBTEST.COM'} at 127.9.101.193:37915
09:04:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.159318  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/wal
09:04:00.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/logs
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data/info.pb
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.194:0
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.101.194
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.194
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:00.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.194
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:00.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:04:00.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:00.598627 10130 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:04:00.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:00.599241 10130 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:00.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:00.600045 10130 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:04:00.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:00.660180 10130 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:04:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:00.663094 10130 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.101.194
09:04:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.736354 10130 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/logs
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/wal
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.194
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.194:0
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data/info.pb
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.194
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.101.194
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:04:00.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.738334 10130 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:00.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:00.739702 10130 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:04:00.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:00.757661 10137 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:01.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:01.135761 10115 heartbeater.cc:500] Master 127.9.101.252:44553 was elected leader, sending a full tablet report...
09:04:02.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:02 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992642, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:02.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.713675  9844 init.cc:295] Successfully reacquired a new kerberos TGT
09:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:02.169961 10136 debug-util.cc:398] Leaking SignalData structure 0x7b080002efa0 after lost signal to thread 10130
09:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:02.883149 10136 kernel_stack_watchdog.cc:198] Thread 10130 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 397ms:
09:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:04:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:00.759006 10138 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:02.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:02.884433 10130 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.125s	user 0.637s	sys 1.281s
09:04:02.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:02.884794 10130 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.125s	user 0.637s	sys 1.281s
09:04:02.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:02.886591 10141 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:02.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:02.890607 10139 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2124 milliseconds
09:04:02.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.890677 10130 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:04:02.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.892067 10130 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:02.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.894631 10130 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:02.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.896013 10130 hybrid_clock.cc:648] HybridClock initialized: now 1794992642895977 us; error 47 us; skew 500 ppm
09:04:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.897068 10130 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:02.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:02 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992642, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:02.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.906804 10130 init.cc:377] Logged in from keytab as kudu/127.9.101.194@KRBTEST.COM (short username kudu)
09:04:02.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.919047 10130 webserver.cc:577] Webserver started at http://127.9.101.194:35909/ using document root <none> and password file <none>
09:04:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.920415 10130 fs_manager.cc:362] Metadata directory not provided
09:04:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.920709 10130 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.921339 10130 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:04:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.929060 10130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data/instance:
09:04:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f425688ab204a438a8b77607a00ab0d"
09:04:02.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:04:02 on dist-test-slave-mh3s"
09:04:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.930784 10130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/wal/instance:
09:04:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f425688ab204a438a8b77607a00ab0d"
09:04:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:04:02 on dist-test-slave-mh3s"
09:04:02.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.947045 10130 fs_manager.cc:696] Time spent creating directory manager: real 0.015s	user 0.009s	sys 0.005s
09:04:02.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.960057 10149 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:04:02.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.962276 10130 fs_manager.cc:730] Time spent opening block manager: real 0.009s	user 0.006s	sys 0.002s
09:04:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.962702 10130 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/wal
09:04:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f425688ab204a438a8b77607a00ab0d"
09:04:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:04:02 on dist-test-slave-mh3s"
09:04:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:02.963201 10130 fs_report.cc:389] FS layout report
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/wal
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/wal
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data/data
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.053648 10130 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:04:03.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.061909 10130 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:03.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.062690 10130 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:03.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.068254 10130 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:04:03.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.075440 10130 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.075748 10130 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
09:04:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.076110 10130 ts_tablet_manager.cc:662] Registered 0 tablets
09:04:03.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.076344 10130 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
09:04:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:03.454909  9805 debug-util.cc:398] Leaking SignalData structure 0x7b08000968c0 after lost signal to thread 9734
09:04:03.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:03.456205  9805 debug-util.cc:398] Leaking SignalData structure 0x7b080005cba0 after lost signal to thread 9808
09:04:03.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.523448 10130 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.194:34551
09:04:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.523710 10262 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.194:34551 every 8 connection(s)
09:04:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.528559 10130 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/data/info.pb
09:04:03.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.533943  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 10130
09:04:03.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.535080  9623 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-1/wal/instance
09:04:03.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.9.101.195@KRBTEST.COM; defaulting to no policy
09:04:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:03.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.9.101.195@KRBTEST.COM" created.
09:04:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:03 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992642, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.194@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:04:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:03 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992642, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.194@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:04:03.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:03 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992642, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.194@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:04:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.703925 10265 heartbeater.cc:345] Connected to a master server at 127.9.101.254:43661
09:04:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.704520 10265 heartbeater.cc:462] Registering TS with master...
09:04:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.706521 10264 heartbeater.cc:345] Connected to a master server at 127.9.101.253:38469
09:04:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.706730 10265 heartbeater.cc:508] Master 127.9.101.254:43661 requested a full tablet report, sending...
09:04:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.706881 10264 heartbeater.cc:462] Registering TS with master...
09:04:03.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.707898 10264 heartbeater.cc:508] Master 127.9.101.253:38469 requested a full tablet report, sending...
09:04:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.711024  9688 ts_manager.cc:210] Registered new tserver with Master: 5f425688ab204a438a8b77607a00ab0d (127.9.101.194:34551)
09:04:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.711436  9774 ts_manager.cc:210] Registered new tserver with Master: 5f425688ab204a438a8b77607a00ab0d (127.9.101.194:34551)
09:04:03.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.718891 10263 heartbeater.cc:345] Connected to a master server at 127.9.101.252:44553
09:04:03.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.719231 10263 heartbeater.cc:462] Registering TS with master...
09:04:03.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.720273 10263 heartbeater.cc:508] Master 127.9.101.252:44553 requested a full tablet report, sending...
09:04:03.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.723055  9865 ts_manager.cc:210] Registered new tserver with Master: 5f425688ab204a438a8b77607a00ab0d (127.9.101.194:34551)
09:04:03.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.725287  9865 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.9.101.194@KRBTEST.COM'} at 127.9.101.194:38241
09:04:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:03.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:03.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.9.101.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:03.729801  9623 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.9.101.195: real 0.131s	user 0.003s	sys 0.015s
09:04:03.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.9.101.195@KRBTEST.COM; defaulting to no policy
09:04:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:03.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.9.101.195@KRBTEST.COM" created.
09:04:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
09:04:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.9.101.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab.
09:04:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:03.787039  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/wal
09:04:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/logs
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data/info.pb
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.195:0
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.101.195
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.195
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:03.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.195
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:04:04.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:04.449103 10277 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:04:04.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:04.449733 10277 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:04.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:04.450740 10277 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:04:04.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:04.487643 10277 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:04:04.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:04.488821 10277 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.9.101.195
09:04:04.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:04.545424 10277 tablet_server_runner.cc:78] Tablet server non-default flags:
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/logs
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/wal
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.195
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.195:0
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data/info.pb
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.195
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:04.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.9.101.195
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:04:04.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:04.547139 10277 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:04.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:04.548537 10277 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:04:04.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:04.567747 10284 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:04.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:04.730531 10263 heartbeater.cc:500] Master 127.9.101.252:44553 was elected leader, sending a full tablet report...
09:04:06.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:05.971514 10283 debug-util.cc:398] Leaking SignalData structure 0x7b080002efa0 after lost signal to thread 10277
09:04:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:06.240811 10277 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 1.675s	user 0.601s	sys 0.919s
09:04:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:06.248891 10277 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 1.683s	user 0.601s	sys 0.919s
09:04:06.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:04.568391 10285 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.252203 10277 server_base.cc:1061] running on GCE node
09:04:06.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:06.256669 10295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:06.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.257967 10277 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:06.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.260330 10277 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:06.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.261826 10277 hybrid_clock.cc:648] HybridClock initialized: now 1794992646261749 us; error 82 us; skew 500 ppm
09:04:06.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.262933 10277 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:06.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:06 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992646, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:06.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.269304 10277 init.cc:377] Logged in from keytab as kudu/127.9.101.195@KRBTEST.COM (short username kudu)
09:04:06.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.277351 10277 webserver.cc:577] Webserver started at http://127.9.101.195:40139/ using document root <none> and password file <none>
09:04:06.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.278591 10277 fs_manager.cc:362] Metadata directory not provided
09:04:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.278851 10277 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:06.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.279414 10277 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
09:04:06.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.285446 10277 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data/instance:
09:04:06.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26c885dc86694504a4874dea7dece4a5"
09:04:06.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:04:06 on dist-test-slave-mh3s"
09:04:06.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.286971 10277 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/wal/instance:
09:04:06.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26c885dc86694504a4874dea7dece4a5"
09:04:06.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:04:06 on dist-test-slave-mh3s"
09:04:06.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.295946 10277 fs_manager.cc:696] Time spent creating directory manager: real 0.008s	user 0.004s	sys 0.004s
09:04:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.302189 10301 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:04:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.303761 10277 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.001s
09:04:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.304150 10277 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/wal
09:04:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26c885dc86694504a4874dea7dece4a5"
09:04:06.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:04:06 on dist-test-slave-mh3s"
09:04:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.304593 10277 fs_report.cc:389] FS layout report
09:04:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/wal
09:04:06.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/wal
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data/data
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.400844 10277 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:04:06.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.416682 10277 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:06.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.417387 10277 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:06.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:06 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992646, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.461541  9667 init.cc:295] Successfully reacquired a new kerberos TGT
09:04:06.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.593508 10277 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
09:04:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.607985 10277 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
09:04:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.608264 10277 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s	user 0.000s	sys 0.000s
09:04:06.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.608620 10277 ts_tablet_manager.cc:662] Registered 0 tablets
09:04:06.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:06.608876 10277 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s	user 0.000s	sys 0.000s
09:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.053925 10277 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.195:43739
09:04:07.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.054216 10414 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.195:43739 every 8 connection(s)
09:04:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.057005 10277 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/data/info.pb
09:04:07.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.062820  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 10277
09:04:07.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.063809  9623 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/ts-2/wal/instance
09:04:07.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:07 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992646, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.195@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:04:07.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:07 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992646, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.195@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:04:07.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:07 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992646, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.195@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:04:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.172966 10415 heartbeater.cc:345] Connected to a master server at 127.9.101.252:44553
09:04:07.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.173527 10415 heartbeater.cc:462] Registering TS with master...
09:04:07.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.175032 10415 heartbeater.cc:508] Master 127.9.101.252:44553 requested a full tablet report, sending...
09:04:07.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.177258 10418 heartbeater.cc:345] Connected to a master server at 127.9.101.254:43661
09:04:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.177615 10418 heartbeater.cc:462] Registering TS with master...
09:04:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.178546  9865 ts_manager.cc:210] Registered new tserver with Master: 26c885dc86694504a4874dea7dece4a5 (127.9.101.195:43739)
09:04:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.178730 10418 heartbeater.cc:508] Master 127.9.101.254:43661 requested a full tablet report, sending...
09:04:07.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.180262  9865 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.9.101.195@KRBTEST.COM'} at 127.9.101.195:49115
09:04:07.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.183436  9688 ts_manager.cc:210] Registered new tserver with Master: 26c885dc86694504a4874dea7dece4a5 (127.9.101.195:43739)
09:04:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.188954 10416 heartbeater.cc:345] Connected to a master server at 127.9.101.253:38469
09:04:07.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.189477 10416 heartbeater.cc:462] Registering TS with master...
09:04:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.190455 10416 heartbeater.cc:508] Master 127.9.101.253:38469 requested a full tablet report, sending...
09:04:07.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.193997  9774 ts_manager.cc:210] Registered new tserver with Master: 26c885dc86694504a4874dea7dece4a5 (127.9.101.195:43739)
09:04:07.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.213125  9623 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
09:04:07.240 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:07.241 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}

09:04:07.251 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
  env_vars {
    key: "KRB5CCNAME"
    value: "/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc"
  }
  env_vars {
    key: "KRB5_CONFIG"
    value: "/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf"
  }
  env_vars {
    key: "KRB5_KDC_PROFILE"
    value: "/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf"
  }
  env_vars {
    key: "KUDU_ENABLE_KRB5_REALM_FIX"
    value: "yes"
  }
}

09:04:07.252 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}

09:04:07.255 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
  masters {
    id {
      type: MASTER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.101.254"
      port: 43661
    }
    bound_http_address {
      host: "127.9.101.254"
      port: 34349
    }
  }
  masters {
    id {
      type: MASTER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.101.253"
      port: 38469
    }
    bound_http_address {
      host: "127.9.101.253"
      port: 33633
    }
  }
  masters {
    id {
      type: MASTER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.101.252"
      port: 44553
    }
    bound_http_address {
      host: "127.9.101.252"
      port: 41731
    }
  }
}

09:04:07.255 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}

09:04:07.257 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
  tservers {
    id {
      type: TSERVER
      index: 0
    }
    bound_rpc_address {
      host: "127.9.101.193"
      port: 36145
    }
    bound_http_address {
      host: "127.9.101.193"
      port: 36043
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 1
    }
    bound_rpc_address {
      host: "127.9.101.194"
      port: 34551
    }
    bound_http_address {
      host: "127.9.101.194"
      port: 35909
    }
  }
  tservers {
    id {
      type: TSERVER
      index: 2
    }
    bound_rpc_address {
      host: "127.9.101.195"
      port: 43739
    }
    bound_http_address {
      host: "127.9.101.195"
      port: 40139
    }
  }
}

09:04:07.257 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
09:04:07.258 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

09:04:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:07 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992647, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
09:04:07.301 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:07.303 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc
09:04:07.313 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:04:07.314 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc
09:04:07.316 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:04:07.347 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
09:04:07.351 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
09:04:07.351 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
  username: "test-admin"
}

09:04:07.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:07 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992647, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:07.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM: 
09:04:07.391 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:07.391 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc
09:04:07.393 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
09:04:07.394 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.101.254:43661
09:04:07.395 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 0
  }
}

09:04:07.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.396699  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9652
09:04:07.423 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:07.423 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.101.253:38469
09:04:07.424 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 1
  }
}

09:04:07.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.424465  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9733
09:04:07.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:07.455 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.9.101.252:44553
09:04:07.455 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
  id {
    type: MASTER
    index: 2
  }
}

09:04:07.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.456148  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9818
09:04:07.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:07.481 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.101.254:43661
09:04:07.484 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 0
  }
}

09:04:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.486493  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/logs
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/info.pb
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.254:43661
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.254
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34349
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.254
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:04:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:04:07.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:04:07.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.579996 10111 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad320 after lost signal to thread 9981
09:04:07.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.580786 10111 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad860 after lost signal to thread 10114
09:04:07.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.743723 10264 heartbeater.cc:648] Failed to heartbeat to 127.9.101.253:38469 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:04:07.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.852370 10430 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:04:07.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.853184 10430 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:07.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.854394 10430 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:04:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.892596 10430 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:04:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.892887 10430 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:04:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.893211 10430 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:04:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.893424 10430 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.893647 10430 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:04:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.893870 10430 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.894034 10430 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39511/jwks.json
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.957000 10430 master_runner.cc:387] Master server non-default flags:
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/logs
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.254
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:07.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.254:43661
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/info.pb
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.254
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34349
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:07.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.958622 10430 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:07.960018 10430 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:04:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.973479 10437 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:08.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:08.184847 10415 heartbeater.cc:500] Master 127.9.101.252:44553 was elected leader, sending a full tablet report...
09:04:08.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:08.205168 10116 heartbeater.cc:648] Failed to heartbeat to 127.9.101.253:38469 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:04:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:08.221287 10418 heartbeater.cc:648] Failed to heartbeat to 127.9.101.254:43661 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.9.101.254:43661: connect: Connection refused (error 111)
09:04:08.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:08 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992648, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:08.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:08.995458  9997 init.cc:295] Successfully reacquired a new kerberos TGT
09:04:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.978775 10440 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:09.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:07.976704 10438 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:09.816555 10439 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1839 milliseconds
09:04:09.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.816728 10430 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:04:09.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.818462 10430 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:09.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.823324 10430 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.825076 10430 hybrid_clock.cc:648] HybridClock initialized: now 1794992649825039 us; error 47 us; skew 500 ppm
09:04:09.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.826431 10430 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:09 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992649, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:09.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.834962 10430 init.cc:377] Logged in from keytab as kudu/127.9.101.254@KRBTEST.COM (short username kudu)
09:04:09.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.841818 10430 webserver.cc:577] Webserver started at http://127.9.101.254:34349/ using document root <none> and password file <none>
09:04:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.843194 10430 fs_manager.cc:362] Metadata directory not provided
09:04:09.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.843514 10430 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:09.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.856778 10430 fs_manager.cc:714] Time spent opening directory manager: real 0.008s	user 0.008s	sys 0.000s
09:04:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.863341 10466 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:04:09.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.865039 10430 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.002s	sys 0.002s
09:04:09.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.865509 10430 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:04:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d040b101867c474ea9335529606211ff"
09:04:09.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:46 on dist-test-slave-mh3s"
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.868697 10430 fs_report.cc:389] FS layout report
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/wal
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/data
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:09.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.939994 10430 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:04:09.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.949172 10430 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:09.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:09.949827 10430 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:10.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.043414 10430 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.254:43661
09:04:10.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.043440 10517 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.254:43661 every 8 connection(s)
09:04:10.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.048116 10430 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-0/data/info.pb
09:04:10.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.058521  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 10430
09:04:10.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:10.059 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.101.253:38469
09:04:10.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 1
  }
}

09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.061743  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/logs
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/info.pb
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:10.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.253:38469
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.253
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33633
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.253
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:10.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:04:10.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:04:10.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.068100 10518 sys_catalog.cc:263] Verifying existing consensus state
09:04:10.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.080574 10518 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Bootstrap starting.
09:04:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.146777 10518 log.cc:826] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Log is configured to *not* fsync() on all Append() calls
09:04:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.172847 10518 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:04:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.174086 10518 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: Bootstrap complete.
09:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.209613 10518 raft_consensus.cc:360] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:10.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.213009 10518 raft_consensus.cc:748] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d040b101867c474ea9335529606211ff, State: Initialized, Role: FOLLOWER
09:04:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.214177 10518 consensus_queue.cc:261] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:10.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.217377 10523 sys_catalog.cc:455] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:10.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.218310 10523 sys_catalog.cc:458] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: This master's current role is: FOLLOWER
09:04:10.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.219636 10518 sys_catalog.cc:565] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: configured and running, proceeding with master startup.
09:04:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.273698 10536 catalog_manager.cc:1310] Loaded cluster ID: 34808d01116841a5a7aa8fa392bac2de
09:04:10.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.273962 10536 catalog_manager.cc:1603] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: loading cluster ID for follower catalog manager: success
09:04:10.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.292040 10536 catalog_manager.cc:1625] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: acquiring CA information for follower catalog manager: success
09:04:10.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.292670 10418 heartbeater.cc:345] Connected to a master server at 127.9.101.254:43661
09:04:10.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.298046 10536 catalog_manager.cc:1653] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:10.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.299998 10483 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "26c885dc86694504a4874dea7dece4a5" instance_seqno: 1794992646969193) as {username='kudu', principal='kudu/127.9.101.195@KRBTEST.COM'} at 127.9.101.195:34771; Asking this server to re-register.
09:04:10.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.302162 10418 heartbeater.cc:462] Registering TS with master...
09:04:10.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.306497 10482 ts_manager.cc:210] Registered new tserver with Master: 26c885dc86694504a4874dea7dece4a5 (127.9.101.195:43739)
09:04:10.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:10 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992649, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:04:10.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.375137 10120 heartbeater.cc:345] Connected to a master server at 127.9.101.254:43661
09:04:10.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.379518 10483 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "80ea234634674162832af7e36b98a453" instance_seqno: 1794992639797772) as {username='kudu', principal='kudu/127.9.101.193@KRBTEST.COM'} at 127.9.101.193:56357; Asking this server to re-register.
09:04:10.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.382041 10120 heartbeater.cc:462] Registering TS with master...
09:04:10.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.386734 10483 ts_manager.cc:210] Registered new tserver with Master: 80ea234634674162832af7e36b98a453 (127.9.101.193:36145)
09:04:10.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.391309 10430 master_runner.cc:187] Error getting master registration for 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111), 
09:04:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.423830 10430 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:04:10.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.597803 10520 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:04:10.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.598672 10520 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:10.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.600208 10520 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:04:10.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.637648 10520 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:04:10.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.638031 10520 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:04:10.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.638546 10520 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:04:10.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.638950 10520 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:10.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.639411 10520 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:04:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.639845 10520 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.640192 10520 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39511/jwks.json
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.688825 10520 master_runner.cc:387] Master server non-default flags:
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/logs
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.253
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.253:38469
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/info.pb
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.253
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33633
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:04:10.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:04:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:04:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:04:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:04:10.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.690575 10520 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.691978 10520 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:04:10.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.704279 10551 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.901451 10265 heartbeater.cc:345] Connected to a master server at 127.9.101.254:43661
09:04:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.905360 10483 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5f425688ab204a438a8b77607a00ab0d" instance_seqno: 1794992643406707) as {username='kudu', principal='kudu/127.9.101.194@KRBTEST.COM'} at 127.9.101.194:44451; Asking this server to re-register.
09:04:10.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.907408 10265 heartbeater.cc:462] Registering TS with master...
09:04:10.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:10.911098 10483 ts_manager.cc:210] Registered new tserver with Master: 5f425688ab204a438a8b77607a00ab0d (127.9.101.194:34551)
09:04:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:11.151455 10111 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad8e0 after lost signal to thread 9981
09:04:11.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:11.152544 10111 debug-util.cc:398] Leaking SignalData structure 0x7b08000b83a0 after lost signal to thread 10114
09:04:12.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:12.130090 10565 raft_consensus.cc:499] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:12.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:12.153407 10565 raft_consensus.cc:521] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:12.241906 10565 leader_election.cc:290] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b39e81019bee4f5a986f915847561ded (127.9.101.253:38469), 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553)
09:04:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:12.901587 10470 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:04:13.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:13 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992653, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:13.377969  9082 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.8.175.253:42151, user_credentials={real_user=slave}} blocked reactor thread for 66944us
09:04:13.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:13.473709 10145 init.cc:295] Successfully reacquired a new kerberos TGT
09:04:13.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:13.617360 10467 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.9.101.252:44553, user_credentials={real_user=slave}} blocked reactor thread for 256729us
09:04:13.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:13.882910 10467 leader_election.cc:336] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553): Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:04:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:13.670495 10470 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.9.101.253:38469, user_credentials={real_user=slave}} blocked reactor thread for 776788us
09:04:13.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:13.960304 10470 leader_election.cc:336] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b39e81019bee4f5a986f915847561ded (127.9.101.253:38469): Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:04:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:13.976940 10470 leader_election.cc:304] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d040b101867c474ea9335529606211ff; no voters: 80cfa9011505477a9da4db0c6f01017d, b39e81019bee4f5a986f915847561ded
09:04:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:14.002027 10585 raft_consensus.cc:2771] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:04:14.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:14.182411  9221 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.TSHeartbeat -> {remote=127.8.175.254:35127, user_credentials={real_user=slave}} blocked reactor thread for 1.26629e+06us
09:04:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:14.366603  9221 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.TSHeartbeat -> {remote=127.8.175.252:45213, user_credentials={real_user=slave}} blocked reactor thread for 134080us
09:04:14.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:14.425256  9084 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.8.175.254:35127, user_credentials={real_user=slave}} blocked reactor thread for 63754.5us
09:04:15.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.455250 10430 master_runner.cc:187] Error getting master registration for 127.9.101.253:38469: Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111), 
09:04:15.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.455744 10430 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:04:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:12.157833 10550 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 10520
09:04:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:15.936942 10550 kernel_stack_watchdog.cc:198] Thread 10520 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 447ms:
09:04:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:04:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:04:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:04:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:04:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:15.938812 10520 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 5.233s	user 0.733s	sys 1.151s
09:04:15.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:15.939230 10520 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 5.233s	user 0.733s	sys 1.151s
09:04:15.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:10.706472 10552 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:15.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:15.941313 10553 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 5231 milliseconds
09:04:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:15.943085 10554 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:15.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.946286 10520 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
09:04:15.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.948226 10520 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:15.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.952435 10520 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:15.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.954077 10520 hybrid_clock.cc:648] HybridClock initialized: now 1794992655954010 us; error 50 us; skew 500 ppm
09:04:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.955749 10520 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:15 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992655, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.964792 10520 init.cc:377] Logged in from keytab as kudu/127.9.101.253@KRBTEST.COM (short username kudu)
09:04:15.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.972885 10520 webserver.cc:577] Webserver started at http://127.9.101.253:33633/ using document root <none> and password file <none>
09:04:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.974319 10520 fs_manager.cc:362] Metadata directory not provided
09:04:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.974627 10520 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:15.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.987721 10520 fs_manager.cc:714] Time spent opening directory manager: real 0.008s	user 0.006s	sys 0.002s
09:04:15.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.994611 10606 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:04:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.996076 10520 fs_manager.cc:730] Time spent opening block manager: real 0.005s	user 0.000s	sys 0.004s
09:04:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.996492 10520 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:04:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b39e81019bee4f5a986f915847561ded"
09:04:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:49 on dist-test-slave-mh3s"
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:15.998839 10520 fs_report.cc:389] FS layout report
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/wal
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/data
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:15.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:16.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.048187 10611 raft_consensus.cc:499] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:16.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.048651 10611 raft_consensus.cc:521] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:16.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.058224 10611 leader_election.cc:290] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b39e81019bee4f5a986f915847561ded (127.9.101.253:38469), 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553)
09:04:16.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.070147 10470 leader_election.cc:336] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b39e81019bee4f5a986f915847561ded (127.9.101.253:38469): Network error: Client connection negotiation failed: client connection to 127.9.101.253:38469: connect: Connection refused (error 111)
09:04:16.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.071041 10467 leader_election.cc:336] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553): Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:04:16.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.071481 10467 leader_election.cc:304] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d040b101867c474ea9335529606211ff; no voters: 80cfa9011505477a9da4db0c6f01017d, b39e81019bee4f5a986f915847561ded
09:04:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.072229 10611 raft_consensus.cc:2771] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
09:04:16.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.073642 10520 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:04:16.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.080675 10520 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:16.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.081241 10520 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:16.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.132102 10259 debug-util.cc:398] Leaking SignalData structure 0x7b08000b65c0 after lost signal to thread 10132
09:04:16.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.133313 10259 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6e80 after lost signal to thread 10262
09:04:16.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.271766 10520 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.253:38469
09:04:16.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.271813 10661 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.253:38469 every 8 connection(s)
09:04:16.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:16 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992656, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:16.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.277256 10520 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-1/data/info.pb
09:04:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.282050  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 10520
09:04:16.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.275190 10297 init.cc:295] Successfully reacquired a new kerberos TGT
09:04:16.284 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:16.285 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.9.101.252:44553
09:04:16.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
  id {
    type: MASTER
    index: 2
  }
}

09:04:16.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.288293  9623 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/logs
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/info.pb
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
09:04:16.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.252:44553
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.252
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41731
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.252
09:04:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
09:04:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
09:04:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
09:04:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:16.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
09:04:16.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
09:04:16.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.331225 10662 sys_catalog.cc:263] Verifying existing consensus state
09:04:16.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.341552 10662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Bootstrap starting.
09:04:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.413026 10662 log.cc:826] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Log is configured to *not* fsync() on all Append() calls
09:04:16.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.445806 10662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:04:16.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.446925 10662 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Bootstrap complete.
09:04:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.484429 10662 raft_consensus.cc:360] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:16.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.488085 10662 raft_consensus.cc:748] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b39e81019bee4f5a986f915847561ded, State: Initialized, Role: FOLLOWER
09:04:16.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.489392 10662 consensus_queue.cc:261] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:16.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.497978 10666 sys_catalog.cc:455] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.502198 10666 sys_catalog.cc:458] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: This master's current role is: FOLLOWER
09:04:16.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.501327 10662 sys_catalog.cc:565] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: configured and running, proceeding with master startup.
09:04:16.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.600925 10680 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
09:04:16.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:16 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992653, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.194@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:04:16.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.638885 10264 heartbeater.cc:345] Connected to a master server at 127.9.101.253:38469
09:04:16.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.647377 10627 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "5f425688ab204a438a8b77607a00ab0d" instance_seqno: 1794992643406707) as {username='kudu', principal='kudu/127.9.101.194@KRBTEST.COM'} at 127.9.101.194:41785; Asking this server to re-register.
09:04:16.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.650022 10264 heartbeater.cc:462] Registering TS with master...
09:04:16.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.655342 10626 ts_manager.cc:210] Registered new tserver with Master: 5f425688ab204a438a8b77607a00ab0d (127.9.101.194:34551)
09:04:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:16 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992655, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:04:16.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:16 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992655, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:04:16.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.761427 10520 master_runner.cc:187] Error getting master registration for 127.9.101.252:44553: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111), 
09:04:16.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.792842 10520 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:04:16.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.944730 10664 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
09:04:16.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.945746 10664 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
09:04:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.947384 10664 flags.cc:432] Enabled unsafe flag: --never_fsync=true
09:04:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:16 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992656, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.195@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:04:16.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.968076 10416 heartbeater.cc:345] Connected to a master server at 127.9.101.253:38469
09:04:16.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.971187 10627 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "26c885dc86694504a4874dea7dece4a5" instance_seqno: 1794992646969193) as {username='kudu', principal='kudu/127.9.101.195@KRBTEST.COM'} at 127.9.101.195:54757; Asking this server to re-register.
09:04:16.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.972932 10416 heartbeater.cc:462] Registering TS with master...
09:04:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:16.976476 10627 ts_manager.cc:210] Registered new tserver with Master: 26c885dc86694504a4874dea7dece4a5 (127.9.101.195:43739)
09:04:16.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.996516 10664 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
09:04:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.996826 10664 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
09:04:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.997255 10664 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
09:04:17.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.997622 10664 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
09:04:17.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.997959 10664 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
09:04:17.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.998332 10664 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
09:04:17.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:16.998824 10664 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39511/jwks.json
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.054828 10664 master_runner.cc:387] Master server non-default flags:
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/logs
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.9.101.212:38071
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/krb5kdc/kudu.keytab
09:04:17.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.9.101.252
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.9.101.252:44553
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39511/jwks.json
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/info.pb
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.9.101.252
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41731
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
09:04:17.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
09:04:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 18 Nov 2026 08:43:22 UTC on e7f111948823
09:04:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14842
09:04:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
09:04:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.056613 10664 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
09:04:17.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.058029 10664 file_cache.cc:504] Constructed file cache file cache with capacity 419430
09:04:17.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:17.098022 10694 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:17 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992648, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.193@KRBTEST.COM for kudu/127.9.101.253@KRBTEST.COM
09:04:17.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.164944 10116 heartbeater.cc:345] Connected to a master server at 127.9.101.253:38469
09:04:17.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.168329 10627 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "80ea234634674162832af7e36b98a453" instance_seqno: 1794992639797772) as {username='kudu', principal='kudu/127.9.101.193@KRBTEST.COM'} at 127.9.101.193:38757; Asking this server to re-register.
09:04:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.169737 10116 heartbeater.cc:462] Registering TS with master...
09:04:17.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.173223 10627 ts_manager.cc:210] Registered new tserver with Master: 80ea234634674162832af7e36b98a453 (127.9.101.193:36145)
09:04:17.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.630869 10680 catalog_manager.cc:1310] Loaded cluster ID: 34808d01116841a5a7aa8fa392bac2de
09:04:17.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.631258 10680 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: loading cluster ID for follower catalog manager: success
09:04:17.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.654395 10680 catalog_manager.cc:1625] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: acquiring CA information for follower catalog manager: success
09:04:17.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:17.679978 10680 catalog_manager.cc:1653] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:18.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.115329 10704 raft_consensus.cc:499] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
09:04:18.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.115979 10704 raft_consensus.cc:521] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:18.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.128573 10493 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b39e81019bee4f5a986f915847561ded" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d040b101867c474ea9335529606211ff" is_pre_election: true
09:04:18.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.129920 10493 raft_consensus.cc:2490] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b39e81019bee4f5a986f915847561ded in term 1.
09:04:18.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.131449 10608 leader_election.cc:304] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b39e81019bee4f5a986f915847561ded, d040b101867c474ea9335529606211ff; no voters: 
09:04:18.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.143108 10706 raft_consensus.cc:2826] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Leader pre-election won for term 2
09:04:18.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.143309 10704 leader_election.cc:290] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d040b101867c474ea9335529606211ff (127.9.101.254:43661), 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553)
09:04:18.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.143569 10706 raft_consensus.cc:499] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
09:04:18.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.143985 10706 raft_consensus.cc:3096] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 1 FOLLOWER]: Advancing to term 2
09:04:18.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.167368 10706 raft_consensus.cc:521] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:18.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.169858 10706 leader_election.cc:290] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [CANDIDATE]: Term 2 election: Requested vote from peers d040b101867c474ea9335529606211ff (127.9.101.254:43661), 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553)
09:04:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.171571 10493 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b39e81019bee4f5a986f915847561ded" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d040b101867c474ea9335529606211ff"
09:04:18.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.172199 10493 raft_consensus.cc:3096] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 1 FOLLOWER]: Advancing to term 2
09:04:18.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.185210 10493 raft_consensus.cc:2490] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b39e81019bee4f5a986f915847561ded in term 2.
09:04:18.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:18.186182 10607 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:04:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.186553 10608 leader_election.cc:304] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: b39e81019bee4f5a986f915847561ded, d040b101867c474ea9335529606211ff; no voters: 
09:04:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.187328 10706 raft_consensus.cc:2826] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 2 FOLLOWER]: Leader election won for term 2
09:04:18.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.187866 10706 raft_consensus.cc:705] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [term 2 LEADER]: Becoming Leader. State: Replica: b39e81019bee4f5a986f915847561ded, State: Running, Role: LEADER
09:04:18.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.188958 10706 consensus_queue.cc:238] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:18.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.224472 10704 sys_catalog.cc:455] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: SysCatalogTable state changed. Reason: New leader b39e81019bee4f5a986f915847561ded. Latest consensus state: current_term: 2 leader_uuid: "b39e81019bee4f5a986f915847561ded" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:18.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.225311 10704 sys_catalog.cc:458] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: This master's current role is: LEADER
09:04:18.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:18.232399 10607 leader_election.cc:336] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553): Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:04:18.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.238821 10710 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
09:04:18.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:18.238467 10607 leader_election.cc:336] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553): Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111)
09:04:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:18.266626 10607 consensus_peers.cc:597] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded -> Peer 80cfa9011505477a9da4db0c6f01017d (127.9.101.252:44553): Couldn't send request to peer 80cfa9011505477a9da4db0c6f01017d. Status: Network error: Client connection negotiation failed: client connection to 127.9.101.252:44553: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
09:04:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.271505 10493 raft_consensus.cc:1285] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [term 2 FOLLOWER]: Refusing update from remote peer b39e81019bee4f5a986f915847561ded: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
09:04:18.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.273664 10704 consensus_queue.cc:1049] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [LEADER]: Connected to new peer: Peer: permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
09:04:18.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.296069 10712 sys_catalog.cc:455] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: SysCatalogTable state changed. Reason: New leader b39e81019bee4f5a986f915847561ded. Latest consensus state: current_term: 2 leader_uuid: "b39e81019bee4f5a986f915847561ded" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:18.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.296864 10712 sys_catalog.cc:458] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: This master's current role is: FOLLOWER
09:04:18.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.300145 10710 catalog_manager.cc:1535] Initializing Kudu cluster ID...
09:04:18.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.326776 10712 sys_catalog.cc:455] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b39e81019bee4f5a986f915847561ded" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:18.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.327520 10712 sys_catalog.cc:458] T 00000000000000000000000000000000 P d040b101867c474ea9335529606211ff [sys.catalog]: This master's current role is: FOLLOWER
09:04:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.334371 10710 catalog_manager.cc:1310] Loaded cluster ID: 34808d01116841a5a7aa8fa392bac2de
09:04:18.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.334720 10710 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
09:04:18.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.360742 10710 catalog_manager.cc:1555] Loading token signing keys...
09:04:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.362478 10704 sys_catalog.cc:455] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b39e81019bee4f5a986f915847561ded" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.363204 10704 sys_catalog.cc:458] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded [sys.catalog]: This master's current role is: LEADER
09:04:18.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.383306 10710 catalog_manager.cc:6232] T 00000000000000000000000000000000 P b39e81019bee4f5a986f915847561ded: Loaded TSK: 0
09:04:18.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:18.385038 10710 catalog_manager.cc:1565] Initializing in-progress tserver states...
09:04:18.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:18 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992658, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.001052  9997 init.cc:295] Successfully reacquired a new kerberos TGT
09:04:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:18.485129 10692 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 10664
09:04:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:19.603673 10692 kernel_stack_watchdog.cc:198] Thread 10664 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 380ms:
09:04:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
09:04:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
09:04:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
09:04:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
09:04:19.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:17.099534 10693 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:19.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:19.605906 10664 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.503s	user 0.635s	sys 1.051s
09:04:19.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:19.606557 10664 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.504s	user 0.635s	sys 1.051s
09:04:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.607098 10664 server_base.cc:1061] running on GCE node
09:04:19.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:19.609561 10696 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
09:04:19.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.611166 10664 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
09:04:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.614496 10664 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
09:04:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.615943 10664 hybrid_clock.cc:648] HybridClock initialized: now 1794992659615865 us; error 87 us; skew 500 ppm
09:04:19.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.617111 10664 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
09:04:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:19 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992659, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:19.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.626236 10664 init.cc:377] Logged in from keytab as kudu/127.9.101.252@KRBTEST.COM (short username kudu)
09:04:19.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.658789 10664 webserver.cc:577] Webserver started at http://127.9.101.252:41731/ using document root <none> and password file <none>
09:04:19.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.660344 10664 fs_manager.cc:362] Metadata directory not provided
09:04:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.660818 10664 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
09:04:19.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.672261 10264 heartbeater.cc:500] Master 127.9.101.253:38469 was elected leader, sending a full tablet report...
09:04:19.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.687446 10664 fs_manager.cc:714] Time spent opening directory manager: real 0.022s	user 0.017s	sys 0.005s
09:04:19.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.701699 10730 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s	user 0.000s	sys 0.000s
09:04:19.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.703307 10664 fs_manager.cc:730] Time spent opening block manager: real 0.012s	user 0.005s	sys 0.001s
09:04:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.703711 10664 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data,/tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:04:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "80cfa9011505477a9da4db0c6f01017d"
09:04:19.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-18 09:03:52 on dist-test-slave-mh3s"
09:04:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.706441 10664 fs_report.cc:389] FS layout report
09:04:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
09:04:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:04:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/wal
09:04:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/data
09:04:19.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
09:04:19.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
09:04:19.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.808908 10664 rpc_server.cc:225] running with OpenSSL 1.1.1  11 Sep 2018
09:04:19.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.817617 10664 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
09:04:19.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.818248 10664 kserver.cc:163] Server-wide thread pool size limit: 3276
09:04:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:19 dist-test-slave-mh3s krb5kdc[9635](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992659, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
09:04:19.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.847383 10462 init.cc:295] Successfully reacquired a new kerberos TGT
09:04:19.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:19.986871 10416 heartbeater.cc:500] Master 127.9.101.253:38469 was elected leader, sending a full tablet report...
09:04:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.086088 10664 rpc_server.cc:307] RPC server started. Bound to: 127.9.101.252:44553
09:04:20.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.086189 10787 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.9.101.252:44553 every 8 connection(s)
09:04:20.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.089295 10664 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskhl46EI/test-tmp/mini-kudu-cluster452828859065745553/master-2/data/info.pb
09:04:20.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.095772  9623 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu as pid 10664
09:04:20.096 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: 
09:04:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.125995 10788 sys_catalog.cc:263] Verifying existing consensus state
09:04:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.136106 10788 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Bootstrap starting.
09:04:20.162 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.101.254:43661(127.9.101.254:43661)] connecting to peer
09:04:20.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.184297 10116 heartbeater.cc:500] Master 127.9.101.253:38469 was elected leader, sending a full tablet report...
09:04:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.210376 10788 log.cc:826] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Log is configured to *not* fsync() on all Append() calls
09:04:20.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.255471 10788 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
09:04:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.256511 10788 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: Bootstrap complete.
09:04:20.259 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.101.253:38469(127.9.101.253:38469)] connecting to peer
09:04:20.271 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.9.101.252:44553(127.9.101.252:44553)] connecting to peer
09:04:20.300 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.9.101.254:43661(127.9.101.254:43661)] Successfully connected to peer
09:04:20.303 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.9.101.252:44553(127.9.101.252:44553)] Successfully connected to peer
09:04:20.304 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.9.101.253:38469(127.9.101.253:38469)] Successfully connected to peer
09:04:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.313515 10788 raft_consensus.cc:360] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:20.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.323705 10788 raft_consensus.cc:748] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 80cfa9011505477a9da4db0c6f01017d, State: Initialized, Role: FOLLOWER
09:04:20.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.324980 10788 consensus_queue.cc:261] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } }
09:04:20.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.366379 10788 sys_catalog.cc:565] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: configured and running, proceeding with master startup.
09:04:20.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.363931 10799 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:20.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.379091 10799 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: This master's current role is: FOLLOWER
09:04:20.462 [WARN - kudu-nio-2] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
09:04:20.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.9.101.253:38469(127.9.101.253:38469)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
09:04:20.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.9.101.253:38469: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
09:04:20.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.9.101.254:43661(127.9.101.254:43661)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
09:04:20.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.9.101.254:43661: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
09:04:20.492 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.9.101.254:43661(127.9.101.254:43661)] handling channelInactive
09:04:20.493 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.9.101.253:38469(127.9.101.253:38469)] handling channelInactive
09:04:20.493 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.9.101.252:44553(127.9.101.252:44553)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
09:04:20.494 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.9.101.252:44553: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
09:04:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:20.490010 10797 negotiation.cc:336] Failed RPC negotiation. Trace:
09:04:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.300645 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:46992 (local address 127.9.101.253:38469)
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.320566 (+ 19921us) server_negotiation.cc:207] Beginning negotiation
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.320596 (+    30us) server_negotiation.cc:400] Waiting for connection header
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.407386 (+ 86790us) server_negotiation.cc:408] Connection header received
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.441201 (+ 33815us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.441235 (+    34us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.441492 (+   257us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.441739 (+   247us) server_negotiation.cc:227] Negotiated authn=SASL
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.489939 (+ 48200us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46992: BlockingRecv error: recv got EOF from 127.0.0.1:46992 (error 108)
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":19223,"thread_start_us":27843,"threads_started":1}
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:20.493057 10796 negotiation.cc:336] Failed RPC negotiation. Trace:
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.298855 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:57836 (local address 127.9.101.254:43661)
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.321957 (+ 23102us) server_negotiation.cc:207] Beginning negotiation
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.321991 (+    34us) server_negotiation.cc:400] Waiting for connection header
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.394820 (+ 72829us) server_negotiation.cc:408] Connection header received
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.440181 (+ 45361us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.440216 (+    35us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.440492 (+   276us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.440744 (+   252us) server_negotiation.cc:227] Negotiated authn=SASL
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.492904 (+ 52160us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57836: BlockingRecv error: recv got EOF from 127.0.0.1:57836 (error 108)
09:04:20.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":22286,"thread_start_us":29749,"threads_started":1}
09:04:20.499 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=396), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.101.254:43661, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.101.253:38469, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.101.252:44553, [373ms] Sub RPC ConnectToMaster: received response from server master-127.9.101.253:38469: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [375ms] Sub RPC ConnectToMaster: received response from server master-127.9.101.254:43661: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [379ms] Sub RPC ConnectToMaster: received response from server master-127.9.101.252:44553: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
	at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
	at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
	at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
	at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
	at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
	at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
	at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
	at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
	at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
	at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
	at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
	at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
	at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
	at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
	at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
	at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
	at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
	at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
	at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
	at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
	at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
	at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:840)

09:04:20.509 [DEBUG - kudu-nio-2] (KuduRpc.java:274) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=408), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.101.254:43661, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.101.253:38469, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.9.101.252:44553, [373ms] Sub RPC ConnectToMaster: received response from server master-127.9.101.253:38469: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [375ms] Sub RPC ConnectToMaster: received response from server master-127.9.101.254:43661: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [379ms] Sub RPC ConnectToMaster: received response from server master-127.9.101.252:44553: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
09:04:20.513 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.101.254:43661(127.9.101.254:43661)] disconnecting while in state TERMINATED
09:04:20.515 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.101.253:38469(127.9.101.253:38469)] disconnecting while in state TERMINATED
09:04:20.515 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.9.101.252:44553(127.9.101.252:44553)] disconnecting while in state TERMINATED
09:04:20.518 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:20.518925 10798 negotiation.cc:336] Failed RPC negotiation. Trace:
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.304605 (+     0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:48196 (local address 127.9.101.252:44553)
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.320760 (+ 16155us) server_negotiation.cc:207] Beginning negotiation
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.320798 (+    38us) server_negotiation.cc:400] Waiting for connection header
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.402266 (+ 81468us) server_negotiation.cc:408] Connection header received
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.441352 (+ 39086us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.441424 (+    72us) server_negotiation.cc:462] Received NEGOTIATE request from client
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.441951 (+   527us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.442510 (+   559us) server_negotiation.cc:227] Negotiated authn=SASL
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1118 09:04:20.518687 (+ 76177us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:48196: BlockingRecv error: recv got EOF from 127.0.0.1:48196 (error 108)
09:04:20.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":15358,"thread_start_us":16002,"threads_started":1}
09:04:20.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:20 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992655, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.253@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:04:20.577 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.9.101.252:44553(127.9.101.252:44553)] handling channelInactive
09:04:20.579 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:04:20.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.583213 10811 catalog_manager.cc:1310] Loaded cluster ID: 34808d01116841a5a7aa8fa392bac2de
09:04:20.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.586778 10811 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: loading cluster ID for follower catalog manager: success
09:04:20.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.589301  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9967
09:04:20.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:20 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992659, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.252@KRBTEST.COM for kudu/127.9.101.254@KRBTEST.COM
09:04:20.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 18 09:04:20 dist-test-slave-mh3s krb5kdc[9635](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794992659, etypes {rep=17 tkt=17 ses=17}, kudu/127.9.101.254@KRBTEST.COM for kudu/127.9.101.252@KRBTEST.COM
09:04:20.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.617339 10811 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: acquiring CA information for follower catalog manager: success
09:04:20.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.632097 10811 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
09:04:20.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.632512 10430 master_runner.cc:233] Existing masters have differing terms: 2 vs 1
09:04:20.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.633074 10430 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
09:04:20.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.650049 10761 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [term 1 FOLLOWER]: Advancing to term 2
09:04:20.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.685133  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 10130
09:04:20.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.704730  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 10277
09:04:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261118 09:04:20.762389 10607 connection.cc:573] server connection from 127.9.101.194:41785 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
09:04:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.772046 10799 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: SysCatalogTable state changed. Reason: New leader b39e81019bee4f5a986f915847561ded. Latest consensus state: current_term: 2 leader_uuid: "b39e81019bee4f5a986f915847561ded" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:20.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.772933 10799 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: This master's current role is: FOLLOWER
09:04:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.775050 10833 sys_catalog.cc:455] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b39e81019bee4f5a986f915847561ded" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d040b101867c474ea9335529606211ff" member_type: VOTER last_known_addr { host: "127.9.101.254" port: 43661 } } peers { permanent_uuid: "b39e81019bee4f5a986f915847561ded" member_type: VOTER last_known_addr { host: "127.9.101.253" port: 38469 } } peers { permanent_uuid: "80cfa9011505477a9da4db0c6f01017d" member_type: VOTER last_known_addr { host: "127.9.101.252" port: 44553 } } }
09:04:20.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.775815 10833 sys_catalog.cc:458] T 00000000000000000000000000000000 P 80cfa9011505477a9da4db0c6f01017d [sys.catalog]: This master's current role is: FOLLOWER
09:04:20.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.782546  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 10430
09:04:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.866784  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 10520
09:04:20.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:20.895946  9623 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 10664
09:04:22.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T09:04:22Z chronyd exiting
09:04:23.179 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
09:04:23.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:23.181576  8895 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9197
09:04:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:23.206398  8895 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9334
09:04:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:23.234536  8895 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9481
09:04:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:23.261718  8895 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 8907
09:04:23.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:23.292971  8895 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 8978
09:04:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261118 09:04:23.324106  8895 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskhl46EI/build/tsan/bin/kudu with pid 9054
09:04:24.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-18T09:04:24Z chronyd exiting
09:04:25.531 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:569)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.lang.Thread.run(Thread.java:840)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
		at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
		at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
		at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
		... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:569)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.lang.Thread.run(Thread.java:840)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
		at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
		at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
		at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
		... 1 more

09:04:25.912 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
09:04:25.912 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:569)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.lang.Thread.run(Thread.java:840)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
		at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
		at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
		at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
		... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.9.101.254:43661,127.9.101.253:38469,127.9.101.252:44553). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
	at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
	at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
	at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
	at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
	at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:569)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
	at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.lang.Thread.run(Thread.java:840)
	Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
		at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
		at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
		at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
		at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
		at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
		at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
		at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
		at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
		at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
		at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
		at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
		at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
		at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
		at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
		at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
		at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
		at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
		at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
		at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
		at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
		at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
		at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
		at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
		at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
		at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
		... 1 more