14:58:03.853 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:58:03.865 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskB842dE/test-tmp
14:58:03.868 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:58:03.871 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskB842dE/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:58:04.191 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:58:04.320 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:58:04.322 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:58:04.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.325345 1355 mini_oidc.cc:150] Starting JWKS server
14:58:04.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.328593 1355 webserver.cc:480] Webserver started at http://127.0.0.1:38555/ using document root <none> and password file <none>
14:58:04.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.329033 1355 mini_oidc.cc:186] Starting OIDC Discovery server
14:58:04.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.330749 1355 webserver.cc:480] Webserver started at http://127.0.0.1:36329/ using document root <none> and password file <none>
14:58:04.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-03T14:58:04Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:58:04.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-03T14:58:04Z Disabled control of system clock
14:58:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.392799 1355 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/wal
14:58:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data
14:58:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/logs
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data/info.pb
14:58:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:04.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.254:32853
14:58:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.254
14:58:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:04.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:04.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:04.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:04.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38555/jwks.json with env {}
14:58:04.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.692523 1367 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:04.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.693080 1367 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:04.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.693732 1367 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:04.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.725878 1367 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:04.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.726184 1367 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:04.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.726390 1367 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:04.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.726626 1367 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:04.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.726857 1367 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:04.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.726996 1367 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38555/jwks.json
14:58:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.763865 1367 master_runner.cc:386] Master server non-default flags:
14:58:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:04.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:04.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/wal
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.254:32853
14:58:04.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38555/jwks.json
14:58:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data/info.pb
14:58:04.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.254
14:58:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/logs
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.765266 1367 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:04.767026 1367 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:04.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.780493 1373 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.184974 1372 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 1367
14:58:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.614682 1372 kernel_stack_watchdog.cc:198] Thread 1367 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
14:58:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:58:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:58:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:58:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:58:06.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:04.781466 1374 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.617761 1367 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.836s user 0.000s sys 0.002s
14:58:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.617795 1375 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1835 milliseconds
14:58:06.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.618172 1367 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.836s user 0.000s sys 0.002s
14:58:06.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.620016 1376 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:06.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.619951 1367 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:06.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.623399 1367 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:06.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.626385 1367 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.627851 1367 hybrid_clock.cc:648] HybridClock initialized: now 1756911486627807 us; error 61 us; skew 500 ppm
14:58:06.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.628665 1367 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.636034 1367 webserver.cc:480] Webserver started at http://127.1.82.254:33539/ using document root <none> and password file <none>
14:58:06.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.636976 1367 fs_manager.cc:362] Metadata directory not provided
14:58:06.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.637177 1367 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:06.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.637642 1367 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.642495 1367 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data/instance:
14:58:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1014081814434d648c13a5f268b1532f"
14:58:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:06 on dist-test-slave-05c6"
14:58:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.643601 1367 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/wal/instance:
14:58:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1014081814434d648c13a5f268b1532f"
14:58:06.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:06 on dist-test-slave-05c6"
14:58:06.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.651315 1367 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
14:58:06.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.658211 1383 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:06.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.659425 1367 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:58:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.659759 1367 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/wal
14:58:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1014081814434d648c13a5f268b1532f"
14:58:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:06 on dist-test-slave-05c6"
14:58:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.660171 1367 fs_report.cc:389] FS layout report
14:58:06.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/wal
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/wal
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data/data
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:06.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:06.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.718648 1367 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.720104 1367 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.720531 1367 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:06.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.804388 1367 rpc_server.cc:307] RPC server started. Bound to: 127.1.82.254:32853
14:58:06.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.804447 1434 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.82.254:32853 every 8 connection(s)
14:58:06.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.807210 1367 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/data/info.pb
14:58:06.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.813483 1435 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:58:06.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.818228 1355 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 1367
14:58:06.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.818792 1355 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-0/wal/instance
14:58:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.824857 1355 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/wal
14:58:06.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/logs
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data/info.pb
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:06.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:06.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:06.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.253:32959
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.253
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:06.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:06.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38555/jwks.json with env {}
14:58:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.829360 1435 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } has no permanent_uuid. Determining permanent_uuid...
14:58:06.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.857761 1435 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } has no permanent_uuid. Determining permanent_uuid...
14:58:06.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.860095 1386 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:06.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.864498 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.253:32959: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:06.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.914115 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } attempt: 1
14:58:06.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.918532 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.253:32959: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:06.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:06.987052 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } attempt: 2
14:58:06.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:06.991505 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.253:32959: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:07.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:07.083078 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } attempt: 3
14:58:07.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.087735 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.253:32959: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:07.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.156836 1437 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:07.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.157465 1437 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:07.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.157955 1437 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.190153 1437 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.190445 1437 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.190649 1437 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:07.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.190829 1437 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:07.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.191016 1437 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:07.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.191161 1437 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38555/jwks.json
14:58:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:07.227830 1437 master_runner.cc:386] Master server non-default flags:
14:58:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data
14:58:07.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/wal
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.253:32959
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:07.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38555/jwks.json
14:58:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data/info.pb
14:58:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.253
14:58:07.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/logs
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:07.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:07.229220 1437 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:07.231184 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } attempt: 4
14:58:07.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:07.231632 1437 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:07.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.238551 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.253:32959: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.247378 1446 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:07.538045 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } attempt: 5
14:58:07.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.551126 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.253:32959: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:08.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.098694 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } attempt: 6
14:58:08.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.133518 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.253:32959: Network error: Client connection negotiation failed: client connection to 127.1.82.253:32959: connect: Connection refused (error 111)
14:58:08.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.649653 1445 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ec0 after lost signal to thread 1437
14:58:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.718792 1445 kernel_stack_watchdog.cc:198] Thread 1437 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
14:58:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:58:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:58:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:58:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:58:08.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:07.248723 1447 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.723928 1437 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.477s user 0.512s sys 0.956s
14:58:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.724342 1437 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.477s user 0.512s sys 0.956s
14:58:08.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.724407 1448 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1471 milliseconds
14:58:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.725793 1449 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:08.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.725867 1437 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.726989 1437 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:08.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.729437 1437 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.730800 1437 hybrid_clock.cc:648] HybridClock initialized: now 1756911488730778 us; error 43 us; skew 500 ppm
14:58:08.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.731925 1437 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:08.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.738425 1437 webserver.cc:480] Webserver started at http://127.1.82.253:46247/ using document root <none> and password file <none>
14:58:08.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.739328 1437 fs_manager.cc:362] Metadata directory not provided
14:58:08.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.739519 1437 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:08.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.739933 1437 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.744462 1437 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data/instance:
14:58:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a"
14:58:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:08 on dist-test-slave-05c6"
14:58:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.745609 1437 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/wal/instance:
14:58:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a"
14:58:08.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:08 on dist-test-slave-05c6"
14:58:08.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.753034 1437 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
14:58:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.759215 1458 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:08.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.760411 1437 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.760738 1437 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/wal
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a"
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:08 on dist-test-slave-05c6"
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.761065 1437 fs_report.cc:389] FS layout report
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/wal
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/wal
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data/data
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:08.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:08.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.824540 1437 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:08.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.826128 1437 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:08.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.826535 1437 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.908772 1437 rpc_server.cc:307] RPC server started. Bound to: 127.1.82.253:32959
14:58:08.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.908857 1509 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.82.253:32959 every 8 connection(s)
14:58:08.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.911729 1437 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/data/info.pb
14:58:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.917845 1510 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:58:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.917865 1355 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 1437
14:58:08.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.918315 1355 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-1/wal/instance
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.920668 1355 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/wal
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:08.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/logs
14:58:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data/info.pb
14:58:08.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:08.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.252:43745
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.252
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:08.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:08.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:08.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38555/jwks.json with env {}
14:58:08.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.934525 1510 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } has no permanent_uuid. Determining permanent_uuid...
14:58:08.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.961854 1510 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } has no permanent_uuid. Determining permanent_uuid...
14:58:08.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:08.976672 1510 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } has no permanent_uuid. Determining permanent_uuid...
14:58:08.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.979040 1462 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:08.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:08.984441 1510 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.034034 1510 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 1
14:58:09.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.038257 1510 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.106750 1510 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 2
14:58:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.111156 1510 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.194206 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } attempt: 7
14:58:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.202569 1510 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 3
14:58:09.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.207686 1435 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } has no permanent_uuid. Determining permanent_uuid...
14:58:09.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.208019 1510 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.213759 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.266947 1512 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.267504 1512 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:09.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.267982 1512 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:09.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.272156 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 1
14:58:09.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.276274 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.301559 1512 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.301875 1512 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.302068 1512 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.302248 1512 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.302436 1512 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:09.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.302582 1512 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38555/jwks.json
14:58:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.347918 1512 master_runner.cc:386] Master server non-default flags:
14:58:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data
14:58:09.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/wal
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.252:43745
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38555/jwks.json
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data/info.pb
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.252
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/logs
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:09.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:09.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.349359 1512 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.351651 1512 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:09.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.351483 1510 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 4
14:58:09.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.355733 1510 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.357645 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 2
14:58:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.362919 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.368571 1525 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:09.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.448361 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 3
14:58:09.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.456324 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.596998 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 4
14:58:09.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.604228 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.655164 1510 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 5
14:58:09.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.662123 1510 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:09.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:09.887897 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 5
14:58:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.897083 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:10.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.209741 1510 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 6
14:58:10.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:10.221550 1510 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:10.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.449756 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 6
14:58:10.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:10.460451 1435 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.1.82.252:43745: Network error: Client connection negotiation failed: client connection to 127.1.82.252:43745: connect: Connection refused (error 111)
14:58:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.369855 1524 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:10.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:09.372355 1527 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:10.616109 1526 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1243 milliseconds
14:58:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.616227 1512 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:10.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.617449 1512 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:10.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.619951 1512 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:10.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.621326 1512 hybrid_clock.cc:648] HybridClock initialized: now 1756911490621263 us; error 72 us; skew 500 ppm
14:58:10.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.622084 1512 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:10.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.629249 1512 webserver.cc:480] Webserver started at http://127.1.82.252:43601/ using document root <none> and password file <none>
14:58:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.630290 1512 fs_manager.cc:362] Metadata directory not provided
14:58:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.630497 1512 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:10.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.630892 1512 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:10.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.635535 1512 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data/instance:
14:58:10.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "041efcecb98a4e1883b6b05602d6f794"
14:58:10.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:10 on dist-test-slave-05c6"
14:58:10.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.636682 1512 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/wal/instance:
14:58:10.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "041efcecb98a4e1883b6b05602d6f794"
14:58:10.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:10 on dist-test-slave-05c6"
14:58:10.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.644268 1512 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
14:58:10.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.650533 1538 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.651762 1512 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.652086 1512 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/wal
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "041efcecb98a4e1883b6b05602d6f794"
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:10 on dist-test-slave-05c6"
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.652390 1512 fs_report.cc:389] FS layout report
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/wal
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/wal
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data/data
14:58:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:10.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:10.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.706215 1512 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.708560 1512 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.709087 1512 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.791543 1512 rpc_server.cc:307] RPC server started. Bound to: 127.1.82.252:43745
14:58:10.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.791604 1589 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.82.252:43745 every 8 connection(s)
14:58:10.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.794387 1512 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/data/info.pb
14:58:10.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.800746 1590 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:58:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.804044 1355 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 1512
14:58:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.804476 1355 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/master-2/wal/instance
14:58:10.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.822813 1590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } has no permanent_uuid. Determining permanent_uuid...
14:58:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.854259 1590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } has no permanent_uuid. Determining permanent_uuid...
14:58:10.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.872927 1590 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } has no permanent_uuid. Determining permanent_uuid...
14:58:10.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.890074 1590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794: Bootstrap starting.
14:58:10.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.897163 1590 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794: Neither blocks nor log segments found. Creating new log.
14:58:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.899482 1590 log.cc:826] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794: Log is configured to *not* fsync() on all Append() calls
14:58:10.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.905707 1590 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794: No bootstrap required, opened a new log
14:58:10.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.927196 1590 raft_consensus.cc:357] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:10.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.928129 1590 raft_consensus.cc:383] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:58:10.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.928426 1590 raft_consensus.cc:738] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 041efcecb98a4e1883b6b05602d6f794, State: Initialized, Role: FOLLOWER
14:58:10.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.929379 1590 consensus_queue.cc:260] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.932770 1599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.933540 1599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: This master's current role is: FOLLOWER
14:58:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.935266 1590 sys_catalog.cc:564] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: configured and running, proceeding with master startup.
14:58:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:10.964533 1544 tablet.cc:2378] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:58:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:10.966461 1610 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:58:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:10.966802 1610 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:58:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.958107 1512 master_runner.cc:186] Error getting master registration for 127.1.82.254:32853: OK, instance_id { permanent_uuid: "1014081814434d648c13a5f268b1532f" instance_seqno: 1756911486781413 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:58:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:10.983743 1512 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:58:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.282140 1510 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 7
14:58:11.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.300835 1510 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: Bootstrap starting.
14:58:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.307021 1510 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: Neither blocks nor log segments found. Creating new log.
14:58:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.310169 1510 log.cc:826] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: Log is configured to *not* fsync() on all Append() calls
14:58:11.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.315822 1510 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: No bootstrap required, opened a new log
14:58:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.334474 1510 raft_consensus.cc:357] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.335141 1510 raft_consensus.cc:383] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:58:11.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.335417 1510 raft_consensus.cc:738] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c33e02674d4b4fcf8e9ce3d5f806eb6a, State: Initialized, Role: FOLLOWER
14:58:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.336371 1510 consensus_queue.cc:260] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:11.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.339828 1614 sys_catalog.cc:455] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.342348 1614 sys_catalog.cc:458] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [sys.catalog]: This master's current role is: FOLLOWER
14:58:11.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.341970 1510 sys_catalog.cc:564] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [sys.catalog]: configured and running, proceeding with master startup.
14:58:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:11.387390 1625 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:58:11.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:11.388442 1625 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:58:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.379463 1437 master_runner.cc:186] Error getting master registration for 127.1.82.254:32853: OK, instance_id { permanent_uuid: "1014081814434d648c13a5f268b1532f" instance_seqno: 1756911486781413 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:58:11.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.411270 1437 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:58:11.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.426672 1599 raft_consensus.cc:491] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:58:11.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.427302 1599 raft_consensus.cc:513] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:11.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.430011 1599 leader_election.cc:290] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1014081814434d648c13a5f268b1532f (127.1.82.254:32853), c33e02674d4b4fcf8e9ce3d5f806eb6a (127.1.82.253:32959)
14:58:11.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.432478 1485 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "041efcecb98a4e1883b6b05602d6f794" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" is_pre_election: true
14:58:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.433466 1485 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 041efcecb98a4e1883b6b05602d6f794 in term 0.
14:58:11.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.435146 1541 leader_election.cc:304] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 041efcecb98a4e1883b6b05602d6f794, c33e02674d4b4fcf8e9ce3d5f806eb6a; no voters:
14:58:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.436461 1599 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:58:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.436975 1599 raft_consensus.cc:491] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:58:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.437404 1599 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 0 FOLLOWER]: Advancing to term 1
14:58:11.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.445966 1599 raft_consensus.cc:513] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:11.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.448895 1599 leader_election.cc:290] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [CANDIDATE]: Term 1 election: Requested vote from peers 1014081814434d648c13a5f268b1532f (127.1.82.254:32853), c33e02674d4b4fcf8e9ce3d5f806eb6a (127.1.82.253:32959)
14:58:11.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.452473 1485 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "041efcecb98a4e1883b6b05602d6f794" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a"
14:58:11.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.453084 1485 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [term 0 FOLLOWER]: Advancing to term 1
14:58:11.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.462008 1485 raft_consensus.cc:2466] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 041efcecb98a4e1883b6b05602d6f794 in term 1.
14:58:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.463387 1541 leader_election.cc:304] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 041efcecb98a4e1883b6b05602d6f794, c33e02674d4b4fcf8e9ce3d5f806eb6a; no voters:
14:58:11.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.464258 1599 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 1 FOLLOWER]: Leader election won for term 1
14:58:11.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.470053 1599 raft_consensus.cc:695] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [term 1 LEADER]: Becoming Leader. State: Replica: 041efcecb98a4e1883b6b05602d6f794, State: Running, Role: LEADER
14:58:11.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.471487 1599 consensus_queue.cc:237] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.483798 1627 sys_catalog.cc:455] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 041efcecb98a4e1883b6b05602d6f794. Latest consensus state: current_term: 1 leader_uuid: "041efcecb98a4e1883b6b05602d6f794" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.484778 1627 sys_catalog.cc:458] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: This master's current role is: LEADER
14:58:11.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.493881 1435 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } attempt: 7
14:58:11.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.500401 1629 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:58:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.508697 1629 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:58:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.536542 1435 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: Bootstrap starting.
14:58:11.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.538959 1485 raft_consensus.cc:1273] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [term 1 FOLLOWER]: Refusing update from remote peer 041efcecb98a4e1883b6b05602d6f794: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:58:11.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.541469 1627 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:58:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.569811 1435 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: Neither blocks nor log segments found. Creating new log.
14:58:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.577723 1435 log.cc:826] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: Log is configured to *not* fsync() on all Append() calls
14:58:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:11.588526 1464 tablet.cc:2378] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:58:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.591487 1614 sys_catalog.cc:455] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 041efcecb98a4e1883b6b05602d6f794. Latest consensus state: current_term: 1 leader_uuid: "041efcecb98a4e1883b6b05602d6f794" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.592556 1614 sys_catalog.cc:458] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [sys.catalog]: This master's current role is: FOLLOWER
14:58:11.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.604976 1435 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: No bootstrap required, opened a new log
14:58:11.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.621918 1614 sys_catalog.cc:455] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "041efcecb98a4e1883b6b05602d6f794" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.622690 1614 sys_catalog.cc:458] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a [sys.catalog]: This master's current role is: FOLLOWER
14:58:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.635046 1435 raft_consensus.cc:357] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.635959 1435 raft_consensus.cc:383] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:58:11.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.636265 1435 raft_consensus.cc:738] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1014081814434d648c13a5f268b1532f, State: Initialized, Role: FOLLOWER
14:58:11.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.637205 1435 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } }
14:58:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.643369 1627 sys_catalog.cc:455] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "041efcecb98a4e1883b6b05602d6f794" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.644109 1627 sys_catalog.cc:458] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: This master's current role is: LEADER
14:58:11.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.649457 1629 catalog_manager.cc:1349] Generated new cluster ID: 1064da380919429ba0f8facc41ce2f01
14:58:11.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.649880 1629 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:58:11.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.651005 1637 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.651638 1637 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [sys.catalog]: This master's current role is: FOLLOWER
14:58:11.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.652949 1435 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [sys.catalog]: configured and running, proceeding with master startup.
14:58:11.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.653568 1409 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "041efcecb98a4e1883b6b05602d6f794" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1014081814434d648c13a5f268b1532f"
14:58:11.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.654330 1409 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [term 0 FOLLOWER]: Advancing to term 1
14:58:11.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.657557 1410 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "041efcecb98a4e1883b6b05602d6f794" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1014081814434d648c13a5f268b1532f" is_pre_election: true
14:58:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.668632 1409 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 041efcecb98a4e1883b6b05602d6f794 in term 1.
14:58:11.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.671329 1408 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [term 1 FOLLOWER]: Refusing update from remote peer 041efcecb98a4e1883b6b05602d6f794: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:58:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.683607 1599 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:58:11.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.719080 1629 catalog_manager.cc:1372] Generated new certificate authority record
14:58:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.721148 1629 catalog_manager.cc:1506] Loading token signing keys...
14:58:11.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:11.732308 1389 tablet.cc:2378] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:58:11.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.778756 1629 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794: Generated new TSK 0
14:58:11.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.780228 1629 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:58:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.795567 1599 sys_catalog.cc:455] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "041efcecb98a4e1883b6b05602d6f794" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.796355 1599 sys_catalog.cc:458] T 00000000000000000000000000000000 P 041efcecb98a4e1883b6b05602d6f794 [sys.catalog]: This master's current role is: LEADER
14:58:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.799664 1637 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 041efcecb98a4e1883b6b05602d6f794. Latest consensus state: current_term: 1 leader_uuid: "041efcecb98a4e1883b6b05602d6f794" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.800451 1637 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [sys.catalog]: This master's current role is: FOLLOWER
14:58:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.799773 1650 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "041efcecb98a4e1883b6b05602d6f794" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1014081814434d648c13a5f268b1532f" member_type: VOTER last_known_addr { host: "127.1.82.254" port: 32853 } } peers { permanent_uuid: "c33e02674d4b4fcf8e9ce3d5f806eb6a" member_type: VOTER last_known_addr { host: "127.1.82.253" port: 32959 } } peers { permanent_uuid: "041efcecb98a4e1883b6b05602d6f794" member_type: VOTER last_known_addr { host: "127.1.82.252" port: 43745 } } }
14:58:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.801440 1650 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f [sys.catalog]: This master's current role is: FOLLOWER
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.827663 1355 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/wal
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/logs
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data/info.pb
14:58:11.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:11.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.193:0
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.82.193
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.193
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:11.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:58:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.851805 1652 catalog_manager.cc:1261] Loaded cluster ID: 1064da380919429ba0f8facc41ce2f01
14:58:11.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.852470 1652 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: loading cluster ID for follower catalog manager: success
14:58:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.871902 1652 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: acquiring CA information for follower catalog manager: success
14:58:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:11.878038 1652 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 1014081814434d648c13a5f268b1532f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:58:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:12.396613 1625 catalog_manager.cc:1261] Loaded cluster ID: 1064da380919429ba0f8facc41ce2f01
14:58:12.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:12.397483 1625 catalog_manager.cc:1554] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: loading cluster ID for follower catalog manager: success
14:58:12.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:12.404996 1625 catalog_manager.cc:1576] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: acquiring CA information for follower catalog manager: success
14:58:12.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:12.417971 1625 catalog_manager.cc:1604] T 00000000000000000000000000000000 P c33e02674d4b4fcf8e9ce3d5f806eb6a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:58:12.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:12.459559 1655 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:12.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:12.460681 1655 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:12.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:12.461774 1655 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:12.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:12.522449 1655 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:12.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:12.524294 1655 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.82.193
14:58:12.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:12.591415 1655 tablet_server_runner.cc:78] Tablet server non-default flags:
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/wal
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.193:0
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data/info.pb
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.193
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:12.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.82.193
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/logs
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:12.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:12.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:12.596300 1655 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:12.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:12.598836 1655 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:12.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:12.629585 1661 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:12.636515 1662 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:15.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:14.021960 1660 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 1655
14:58:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.110376 1655 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.476s user 0.774s sys 1.153s
14:58:15.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.112033 1655 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.478s user 0.774s sys 1.153s
14:58:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.124677 1655 server_base.cc:1047] running on GCE node
14:58:15.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.124769 1668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:15.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.126601 1655 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:15.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.129249 1655 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.130741 1655 hybrid_clock.cc:648] HybridClock initialized: now 1756911495130611 us; error 144 us; skew 500 ppm
14:58:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.131754 1655 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.140118 1655 webserver.cc:480] Webserver started at http://127.1.82.193:37311/ using document root <none> and password file <none>
14:58:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.141073 1655 fs_manager.cc:362] Metadata directory not provided
14:58:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.141269 1655 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.141896 1655 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:15.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.147799 1655 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data/instance:
14:58:15.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60630c49424c4e01adb3d5aa7adaedad"
14:58:15.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:15 on dist-test-slave-05c6"
14:58:15.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.149003 1655 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/wal/instance:
14:58:15.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60630c49424c4e01adb3d5aa7adaedad"
14:58:15.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:15 on dist-test-slave-05c6"
14:58:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.157088 1655 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
14:58:15.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.163614 1673 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.164806 1655 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:58:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.165117 1655 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/wal
14:58:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "60630c49424c4e01adb3d5aa7adaedad"
14:58:15.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:15 on dist-test-slave-05c6"
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.165485 1655 fs_report.cc:389] FS layout report
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/wal
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/wal
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data/data
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:15.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.227763 1655 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:15.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.229367 1655 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.229933 1655 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.234026 1655 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:58:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.239217 1655 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:58:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.239461 1655 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:58:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.239740 1655 ts_tablet_manager.cc:610] Registered 0 tablets
14:58:15.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.239957 1655 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:58:15.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.451485 1655 rpc_server.cc:307] RPC server started. Bound to: 127.1.82.193:46229
14:58:15.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.451535 1785 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.82.193:46229 every 8 connection(s)
14:58:15.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.455920 1655 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/data/info.pb
14:58:15.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.460309 1355 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 1655
14:58:15.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.460988 1355 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-0/wal/instance
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.503492 1355 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/wal
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/logs
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data/info.pb
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.194:0
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.82.194
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.194
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:15.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:58:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.539752 1787 heartbeater.cc:344] Connected to a master server at 127.1.82.253:32959
14:58:15.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.540277 1787 heartbeater.cc:461] Registering TS with master...
14:58:15.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.541857 1787 heartbeater.cc:507] Master 127.1.82.253:32959 requested a full tablet report, sending...
14:58:15.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.541988 1786 heartbeater.cc:344] Connected to a master server at 127.1.82.252:43745
14:58:15.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.542299 1786 heartbeater.cc:461] Registering TS with master...
14:58:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.543171 1786 heartbeater.cc:507] Master 127.1.82.252:43745 requested a full tablet report, sending...
14:58:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.543793 1788 heartbeater.cc:344] Connected to a master server at 127.1.82.254:32853
14:58:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.544134 1788 heartbeater.cc:461] Registering TS with master...
14:58:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.544978 1788 heartbeater.cc:507] Master 127.1.82.254:32853 requested a full tablet report, sending...
14:58:15.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.545473 1475 ts_manager.cc:194] Registered new tserver with Master: 60630c49424c4e01adb3d5aa7adaedad (127.1.82.193:46229)
14:58:15.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.547634 1555 ts_manager.cc:194] Registered new tserver with Master: 60630c49424c4e01adb3d5aa7adaedad (127.1.82.193:46229)
14:58:15.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.548666 1400 ts_manager.cc:194] Registered new tserver with Master: 60630c49424c4e01adb3d5aa7adaedad (127.1.82.193:46229)
14:58:15.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.550635 1555 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.82.193:54739
14:58:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.851496 1796 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.851999 1796 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:15.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.852447 1796 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:15.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.886152 1796 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:15.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.886977 1796 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.82.194
14:58:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.930097 1796 tablet_server_runner.cc:78] Tablet server non-default flags:
14:58:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:15.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/wal
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.194:0
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data/info.pb
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.194
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.82.194
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/logs
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.931841 1796 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:15.934268 1796 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.966809 1802 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:16.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:16.555935 1786 heartbeater.cc:499] Master 127.1.82.252:43745 was elected leader, sending a full tablet report...
14:58:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:15.967983 1803 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:17.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.216197 1796 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.252s user 0.494s sys 0.755s
14:58:17.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.216696 1796 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.253s user 0.494s sys 0.755s
14:58:17.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.221712 1796 server_base.cc:1047] running on GCE node
14:58:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.223448 1808 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:17.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.224969 1796 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:17.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.227866 1796 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:17.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.229637 1796 hybrid_clock.cc:648] HybridClock initialized: now 1756911497229571 us; error 63 us; skew 500 ppm
14:58:17.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.230867 1796 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:17.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.240238 1796 webserver.cc:480] Webserver started at http://127.1.82.194:43143/ using document root <none> and password file <none>
14:58:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.241711 1796 fs_manager.cc:362] Metadata directory not provided
14:58:17.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.242020 1796 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:17.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.242604 1796 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.250453 1796 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data/instance:
14:58:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99594091547b431eab24565878a920ce"
14:58:17.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:17 on dist-test-slave-05c6"
14:58:17.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.252193 1796 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/wal/instance:
14:58:17.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99594091547b431eab24565878a920ce"
14:58:17.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:17 on dist-test-slave-05c6"
14:58:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.263091 1796 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.012s sys 0.000s
14:58:17.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.271423 1813 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:17.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.272655 1796 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:58:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.272995 1796 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/wal
14:58:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99594091547b431eab24565878a920ce"
14:58:17.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:17 on dist-test-slave-05c6"
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.273362 1796 fs_report.cc:389] FS layout report
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/wal
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/wal
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data/data
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:17.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:17.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.356014 1796 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:17.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.357532 1796 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:17.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.357950 1796 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:17.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.360467 1796 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:58:17.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.364791 1796 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:58:17.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.365005 1796 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:58:17.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.365208 1796 ts_tablet_manager.cc:610] Registered 0 tablets
14:58:17.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.365372 1796 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
14:58:17.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.524528 1796 rpc_server.cc:307] RPC server started. Bound to: 127.1.82.194:34677
14:58:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.524628 1925 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.82.194:34677 every 8 connection(s)
14:58:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.527081 1796 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/data/info.pb
14:58:17.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.529888 1355 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 1796
14:58:17.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.530395 1355 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-1/wal/instance
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.556694 1355 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/wal
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:17.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/logs
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data/info.pb
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.195:0
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.82.195
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.195
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:17.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:58:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.612370 1926 heartbeater.cc:344] Connected to a master server at 127.1.82.252:43745
14:58:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.612586 1928 heartbeater.cc:344] Connected to a master server at 127.1.82.254:32853
14:58:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.612933 1926 heartbeater.cc:461] Registering TS with master...
14:58:17.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.613003 1928 heartbeater.cc:461] Registering TS with master...
14:58:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.614543 1926 heartbeater.cc:507] Master 127.1.82.252:43745 requested a full tablet report, sending...
14:58:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.615192 1928 heartbeater.cc:507] Master 127.1.82.254:32853 requested a full tablet report, sending...
14:58:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.616022 1927 heartbeater.cc:344] Connected to a master server at 127.1.82.253:32959
14:58:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.616364 1927 heartbeater.cc:461] Registering TS with master...
14:58:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.617219 1927 heartbeater.cc:507] Master 127.1.82.253:32959 requested a full tablet report, sending...
14:58:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.617915 1400 ts_manager.cc:194] Registered new tserver with Master: 99594091547b431eab24565878a920ce (127.1.82.194:34677)
14:58:17.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.618427 1555 ts_manager.cc:194] Registered new tserver with Master: 99594091547b431eab24565878a920ce (127.1.82.194:34677)
14:58:17.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.620096 1555 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.82.194:50303
14:58:17.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.620929 1475 ts_manager.cc:194] Registered new tserver with Master: 99594091547b431eab24565878a920ce (127.1.82.194:34677)
14:58:17.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.908941 1933 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:17.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.909473 1933 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:17.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.909907 1933 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:17.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.942744 1933 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.943745 1933 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.82.195
14:58:17.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.980377 1933 tablet_server_runner.cc:78] Tablet server non-default flags:
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.82.212:43719
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/wal
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.82.195:0
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data/info.pb
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.82.195
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.82.254:32853,127.1.82.253:32959,127.1.82.252:43745
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:17.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.82.195
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/logs
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.982085 1933 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:17.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:17.983685 1933 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:17.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.996937 1942 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:18.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:18.624435 1926 heartbeater.cc:499] Master 127.1.82.252:43745 was elected leader, sending a full tablet report...
14:58:19.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:17.998948 1943 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:19.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:19.331970 1945 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:19.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:19.334456 1944 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1330 milliseconds
14:58:19.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.334604 1933 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:19.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.335862 1933 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:19.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.338119 1933 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:19.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.339489 1933 hybrid_clock.cc:648] HybridClock initialized: now 1756911499339462 us; error 40 us; skew 500 ppm
14:58:19.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.340319 1933 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.346614 1933 webserver.cc:480] Webserver started at http://127.1.82.195:37445/ using document root <none> and password file <none>
14:58:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.347618 1933 fs_manager.cc:362] Metadata directory not provided
14:58:19.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.347843 1933 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:19.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.348356 1933 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.352747 1933 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data/instance:
14:58:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e2955854aeb4d619a37e5a27a8c1b61"
14:58:19.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:19 on dist-test-slave-05c6"
14:58:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.353958 1933 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/wal/instance:
14:58:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e2955854aeb4d619a37e5a27a8c1b61"
14:58:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:19 on dist-test-slave-05c6"
14:58:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.361471 1933 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
14:58:19.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.367847 1953 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.368975 1933 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:58:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.369313 1933 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/wal
14:58:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e2955854aeb4d619a37e5a27a8c1b61"
14:58:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:19 on dist-test-slave-05c6"
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.369630 1933 fs_report.cc:389] FS layout report
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/wal
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/wal
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data/data
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:19.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.437196 1933 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:19.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.438959 1933 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:19.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.439409 1933 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.441929 1933 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:58:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.446190 1933 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:58:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.446400 1933 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:58:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.446592 1933 ts_tablet_manager.cc:610] Registered 0 tablets
14:58:19.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.446723 1933 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:58:19.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.605549 1933 rpc_server.cc:307] RPC server started. Bound to: 127.1.82.195:36719
14:58:19.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.605633 2065 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.82.195:36719 every 8 connection(s)
14:58:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.608157 1933 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/data/info.pb
14:58:19.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.615334 1355 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 1933
14:58:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.615904 1355 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster5372978903275011770/ts-2/wal/instance
14:58:19.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.659632 2068 heartbeater.cc:344] Connected to a master server at 127.1.82.254:32853
14:58:19.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.660158 2066 heartbeater.cc:344] Connected to a master server at 127.1.82.252:43745
14:58:19.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.660181 2068 heartbeater.cc:461] Registering TS with master...
14:58:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.660519 2066 heartbeater.cc:461] Registering TS with master...
14:58:19.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.661795 2066 heartbeater.cc:507] Master 127.1.82.252:43745 requested a full tablet report, sending...
14:58:19.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.661798 2068 heartbeater.cc:507] Master 127.1.82.254:32853 requested a full tablet report, sending...
14:58:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.662776 2067 heartbeater.cc:344] Connected to a master server at 127.1.82.253:32959
14:58:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.663152 2067 heartbeater.cc:461] Registering TS with master...
14:58:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.664584 2067 heartbeater.cc:507] Master 127.1.82.253:32959 requested a full tablet report, sending...
14:58:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.664978 1400 ts_manager.cc:194] Registered new tserver with Master: 5e2955854aeb4d619a37e5a27a8c1b61 (127.1.82.195:36719)
14:58:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.665169 1555 ts_manager.cc:194] Registered new tserver with Master: 5e2955854aeb4d619a37e5a27a8c1b61 (127.1.82.195:36719)
14:58:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.666815 1555 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.1.82.195:33497
14:58:19.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.666899 1475 ts_manager.cc:194] Registered new tserver with Master: 5e2955854aeb4d619a37e5a27a8c1b61 (127.1.82.195:36719)
14:58:19.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:19.678586 1355 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:58:19.690 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:58:19.692 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:58:19.704 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.82.254"
port: 32853
}
bound_http_address {
host: "127.1.82.254"
port: 33539
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.82.253"
port: 32959
}
bound_http_address {
host: "127.1.82.253"
port: 46247
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.82.252"
port: 43745
}
bound_http_address {
host: "127.1.82.252"
port: 43601
}
}
}
14:58:19.708 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:58:19.710 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.82.193"
port: 46229
}
bound_http_address {
host: "127.1.82.193"
port: 37311
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.82.194"
port: 34677
}
bound_http_address {
host: "127.1.82.194"
port: 43143
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.82.195"
port: 36719
}
bound_http_address {
host: "127.1.82.195"
port: 37445
}
}
}
14:58:19.710 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:58:19.894 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:58:19.920 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:58:20.052 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskB842dE/test-tmp
14:58:20.053 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:58:20.053 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskB842dE/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:58:20.055 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:58:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:20.154857 1922 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac100 after lost signal to thread 1797
14:58:20.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:20.155690 1922 debug-util.cc:398] Leaking SignalData structure 0x7b08000b71a0 after lost signal to thread 1925
14:58:20.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:20.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:58:20.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:20.422757 2077 mini_oidc.cc:150] Starting JWKS server
14:58:20.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:20.427104 2077 webserver.cc:480] Webserver started at http://127.0.0.1:32965/ using document root <none> and password file <none>
14:58:20.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:20.427536 2077 mini_oidc.cc:186] Starting OIDC Discovery server
14:58:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:20.429744 2077 webserver.cc:480] Webserver started at http://127.0.0.1:45069/ using document root <none> and password file <none>
14:58:20.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:58:20.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/principal' for realm 'KRBTEST.COM',
14:58:20.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:58:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:20 dist-test-slave-05c6 krb5kdc[2089](info): setting up network...
14:58:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:58:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:20 dist-test-slave-05c6 krb5kdc[2089](info): set up 2 sockets
14:58:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:20 dist-test-slave-05c6 krb5kdc[2089](info): commencing operation
14:58:20.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:58:20.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:20.670094 2066 heartbeater.cc:499] Master 127.1.82.252:43745 was elected leader, sending a full tablet report...
14:58:22.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:22.608596 2077 mini_kdc.cc:121] Time spent starting KDC: real 2.140s user 0.009s sys 0.013s
14:58:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:58:22.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:58:22.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:58:22.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:58:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:58:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:58:22.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:58:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:58:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:58:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:58:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/test-user.keytab.
14:58:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/test-user.keytab.
14:58:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:22 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911502, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:22.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:58:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-03T14:58:22Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:58:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-03T14:58:22Z Disabled control of system clock
14:58:22.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.7.126@KRBTEST.COM; defaulting to no policy
14:58:22.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:22.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.7.126@KRBTEST.COM" created.
14:58:22.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:22.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:22.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:22.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.7.126@KRBTEST.COM; defaulting to no policy
14:58:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:22.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.7.126@KRBTEST.COM" created.
14:58:22.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:22.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:22.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:22.854040 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data
14:58:22.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/logs
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/info.pb
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.126:43677
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.126
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.126
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:58:22.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.184574 2106 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.185276 2106 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.186110 2106 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.218305 2106 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:58:23.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.218614 2106 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.219048 2106 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.219365 2106 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.219673 2106 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:23.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.219969 2106 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:23.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.220192 2106 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32965/jwks.json
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:23.259649 2106 master_runner.cc:386] Master server non-default flags:
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.126
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.126:43677
14:58:23.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:23.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:23.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:23.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:23.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:23.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/info.pb
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.126
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/logs
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:23.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:23.261322 2106 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:23.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:23.262787 2106 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.275601 2112 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:23.277124 2113 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:24.679023 2111 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2106
14:58:24.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:24.916679 2106 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.641s user 0.525s sys 1.082s
14:58:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:24.918919 2115 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:24.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:24.919004 2106 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.644s user 0.526s sys 1.083s
14:58:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:24.919757 2114 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1641 milliseconds
14:58:24.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.919725 2106 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:24.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.923435 2106 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:24.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.925998 2106 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:24.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.927373 2106 hybrid_clock.cc:648] HybridClock initialized: now 1756911504927322 us; error 65 us; skew 500 ppm
14:58:24.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.928241 2106 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:24 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911504, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:24.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.934446 2106 init.cc:377] Logged in from keytab as kudu/127.2.7.126@KRBTEST.COM (short username kudu)
14:58:24.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.939620 2106 webserver.cc:480] Webserver started at http://127.2.7.126:42535/ using document root <none> and password file <none>
14:58:24.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.940516 2106 fs_manager.cc:362] Metadata directory not provided
14:58:24.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.940719 2106 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:24.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.941095 2106 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:24.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.945600 2106 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/instance:
14:58:24.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca49f183b1bf4edca721f1740b11706a"
14:58:24.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:24 on dist-test-slave-05c6"
14:58:24.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.946727 2106 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal/instance:
14:58:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca49f183b1bf4edca721f1740b11706a"
14:58:24.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:24 on dist-test-slave-05c6"
14:58:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.954144 2106 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
14:58:24.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.959836 2125 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.961048 2106 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
14:58:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.961429 2106 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca49f183b1bf4edca721f1740b11706a"
14:58:24.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:24 on dist-test-slave-05c6"
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:24.961764 2106 fs_report.cc:389] FS layout report
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/data
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:25.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.015271 2106 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.040124 2106 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.040544 2106 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:25.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.125967 2106 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.126:43677
14:58:25.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.126035 2176 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.126:43677 every 8 connection(s)
14:58:25.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.128711 2106 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/info.pb
14:58:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.134842 2177 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:58:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.135165 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2106
14:58:25.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.135705 2077 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal/instance
14:58:25.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.146854 2177 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } has no permanent_uuid. Determining permanent_uuid...
14:58:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.7.125@KRBTEST.COM; defaulting to no policy
14:58:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.7.125@KRBTEST.COM" created.
14:58:25.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:25 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911504, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.126@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:25.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:25.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.192859 2177 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } has no permanent_uuid. Determining permanent_uuid...
14:58:25.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.195617 2128 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:25.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.7.125@KRBTEST.COM; defaulting to no policy
14:58:25.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.200138 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:25.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.7.125@KRBTEST.COM" created.
14:58:25.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:25.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:25.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.226332 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/logs
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/info.pb
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.125:46609
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.125
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.125
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:58:25.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:25.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.249719 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 1
14:58:25.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.254252 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:25.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.322726 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 2
14:58:25.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.327930 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.419591 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 3
14:58:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.423368 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.551924 2186 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.552474 2186 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.553092 2186 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:25.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.566833 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 4
14:58:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.571735 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.586879 2186 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:58:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.587168 2186 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:25.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.587555 2186 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.587878 2186 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.588179 2186 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.588466 2186 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:25.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.588723 2186 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32965/jwks.json
14:58:25.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.628418 2186 master_runner.cc:386] Master server non-default flags:
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.125
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.125:46609
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/info.pb
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.125
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/logs
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.630134 2186 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:25.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.631321 2186 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:25.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.643203 2192 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:25.871156 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 5
14:58:25.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.878718 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:26.426296 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 6
14:58:26.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:26.481161 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:27.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.045526 2191 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2186
14:58:27.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.515556 2186 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.872s user 0.655s sys 1.157s
14:58:27.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.516431 2186 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.872s user 0.655s sys 1.157s
14:58:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:25.644932 2193 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:27.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.518922 2195 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:27.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.522337 2194 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1873 milliseconds
14:58:27.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.522359 2186 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:27.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.523751 2186 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.526350 2186 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.527750 2186 hybrid_clock.cc:648] HybridClock initialized: now 1756911507527722 us; error 51 us; skew 500 ppm
14:58:27.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.528767 2186 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:27.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:27 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911507, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:27.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.535336 2186 init.cc:377] Logged in from keytab as kudu/127.2.7.125@KRBTEST.COM (short username kudu)
14:58:27.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.540572 2186 webserver.cc:480] Webserver started at http://127.2.7.125:44041/ using document root <none> and password file <none>
14:58:27.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.541571 2186 fs_manager.cc:362] Metadata directory not provided
14:58:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.541831 2186 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.541738 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 7
14:58:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.542433 2186 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.548096 2186 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/instance:
14:58:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc97b3e1a10940c0aad4d42a187be1e5"
14:58:27.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:27 on dist-test-slave-05c6"
14:58:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.549228 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.125:46609: Network error: Client connection negotiation failed: client connection to 127.2.7.125:46609: connect: Connection refused (error 111)
14:58:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.549445 2186 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal/instance:
14:58:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc97b3e1a10940c0aad4d42a187be1e5"
14:58:27.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:27 on dist-test-slave-05c6"
14:58:27.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.557505 2186 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
14:58:27.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.563688 2210 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:27.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.564854 2186 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:58:27.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.565172 2186 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:27.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc97b3e1a10940c0aad4d42a187be1e5"
14:58:27.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:27 on dist-test-slave-05c6"
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.565603 2186 fs_report.cc:389] FS layout report
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/data
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.630577 2186 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:27.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.637140 2186 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.637955 2186 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:27.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.733422 2186 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.125:46609
14:58:27.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.733528 2261 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.125:46609 every 8 connection(s)
14:58:27.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.736294 2186 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/info.pb
14:58:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.741685 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2186
14:58:27.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.742183 2077 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal/instance
14:58:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.742782 2262 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:58:27.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.757467 2262 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } has no permanent_uuid. Determining permanent_uuid...
14:58:27.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.7.124@KRBTEST.COM; defaulting to no policy
14:58:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:27.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.7.124@KRBTEST.COM" created.
14:58:27.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:27 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911507, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.125@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:27.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:27.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:27.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:27.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.798702 2262 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } has no permanent_uuid. Determining permanent_uuid...
14:58:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.7.124@KRBTEST.COM; defaulting to no policy
14:58:27.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:27.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.7.124@KRBTEST.COM" created.
14:58:27.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:27 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911507, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.125@KRBTEST.COM for kudu/127.2.7.125@KRBTEST.COM
14:58:27.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.837049 2262 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } has no permanent_uuid. Determining permanent_uuid...
14:58:27.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.839711 2211 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:27.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:27.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:27.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:27.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.845090 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.847280 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:27.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/logs
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/info.pb
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:27.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.124:35655
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.124
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.124
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:27.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:58:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.894738 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 1
14:58:27.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.899839 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:27.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:27.968286 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 2
14:58:27.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:27.972440 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:28.063831 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 3
14:58:28.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.068424 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:28.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.175186 2272 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.175866 2272 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:28.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.176615 2272 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:28.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.209075 2272 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:58:28.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.209398 2272 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.209784 2272 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.210112 2272 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.210415 2272 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:28.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.210706 2272 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:28.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.210923 2272 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32965/jwks.json
14:58:28.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:28.211828 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 4
14:58:28.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.215750 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:28.250859 2272 master_runner.cc:386] Master server non-default flags:
14:58:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.124
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.124:35655
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/info.pb
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.124
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:28.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/logs
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:28.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:28.252522 2272 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:28.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:28.253919 2272 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.267663 2279 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:28.515159 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 5
14:58:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.525375 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:29.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:29.073097 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 6
14:58:29.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:29.114403 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:29.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:29.639688 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } attempt: 8
14:58:29.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:29 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911504, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.126@KRBTEST.COM for kudu/127.2.7.125@KRBTEST.COM
14:58:29.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:29.706162 2177 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } has no permanent_uuid. Determining permanent_uuid...
14:58:29.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:29.776994 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:29.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:29.842628 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 1
14:58:29.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:29.851207 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:29.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:29.904757 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 2
14:58:29.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:29.914178 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:29.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:29.990744 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 3
14:58:30.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.000383 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:28.268028 2280 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:29.670928 2278 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 2272
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.110860 2278 kernel_stack_watchdog.cc:198] Thread 2272 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.111126 2272 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.843s user 0.639s sys 1.080s
14:58:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.111636 2272 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.844s user 0.639s sys 1.080s
14:58:30.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.114292 2281 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1842 milliseconds
14:58:30.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.115772 2282 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:30.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.115862 2272 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:30.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.117146 2272 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.119762 2272 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:30.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.121147 2272 hybrid_clock.cc:648] HybridClock initialized: now 1756911510121100 us; error 48 us; skew 500 ppm
14:58:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.122053 2272 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911510, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:30.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.128968 2272 init.cc:377] Logged in from keytab as kudu/127.2.7.124@KRBTEST.COM (short username kudu)
14:58:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.135089 2272 webserver.cc:480] Webserver started at http://127.2.7.124:46313/ using document root <none> and password file <none>
14:58:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.136185 2272 fs_manager.cc:362] Metadata directory not provided
14:58:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.136432 2272 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.136969 2272 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.141790 2272 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/instance:
14:58:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7032a980ea84c088d1c1d07dc94a137"
14:58:30.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:30 on dist-test-slave-05c6"
14:58:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.143071 2272 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal/instance:
14:58:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7032a980ea84c088d1c1d07dc94a137"
14:58:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:30 on dist-test-slave-05c6"
14:58:30.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.151932 2272 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.002s
14:58:30.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.155916 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 4
14:58:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.158638 2297 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.159945 2272 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:58:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.160316 2272 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7032a980ea84c088d1c1d07dc94a137"
14:58:30.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:30 on dist-test-slave-05c6"
14:58:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.160914 2177 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.160691 2272 fs_report.cc:389] FS layout report
14:58:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/data
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:30.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:30.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.175024 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 7
14:58:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.183231 2262 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.2.7.124:35655: Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:30.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.229205 2272 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:30.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.239495 2272 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:30.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.239919 2272 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.327915 2272 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.124:35655
14:58:30.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.327991 2350 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.124:35655 every 8 connection(s)
14:58:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.330782 2272 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/info.pb
14:58:30.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.333084 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2272
14:58:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.333712 2077 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal/instance
14:58:30.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.337249 2351 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:58:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.358421 2351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } has no permanent_uuid. Determining permanent_uuid...
14:58:30.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911502, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:30.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911510, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.124@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.400569 2351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } has no permanent_uuid. Determining permanent_uuid...
14:58:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911502, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.7.125@KRBTEST.COM
14:58:30.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911510, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.124@KRBTEST.COM for kudu/127.2.7.125@KRBTEST.COM
14:58:30.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.424268 2351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } has no permanent_uuid. Determining permanent_uuid...
14:58:30.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911502, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.7.124@KRBTEST.COM
14:58:30.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911510, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.124@KRBTEST.COM for kudu/127.2.7.124@KRBTEST.COM
14:58:30.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.456691 2351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Bootstrap starting.
14:58:30.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.457338 2177 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 5
14:58:30.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.465517 2351 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Neither blocks nor log segments found. Creating new log.
14:58:30.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.468006 2351 log.cc:826] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Log is configured to *not* fsync() on all Append() calls
14:58:30.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:30 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911504, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.126@KRBTEST.COM for kudu/127.2.7.124@KRBTEST.COM
14:58:30.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.474400 2351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: No bootstrap required, opened a new log
14:58:30.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.490932 2177 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Bootstrap starting.
14:58:30.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.497622 2351 raft_consensus.cc:357] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:30.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.498302 2351 raft_consensus.cc:383] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:58:30.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.498582 2351 raft_consensus.cc:738] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d7032a980ea84c088d1c1d07dc94a137, State: Initialized, Role: FOLLOWER
14:58:30.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.499150 2177 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Neither blocks nor log segments found. Creating new log.
14:58:30.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.499544 2351 consensus_queue.cc:260] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.500998 2177 log.cc:826] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Log is configured to *not* fsync() on all Append() calls
14:58:30.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.503400 2363 sys_catalog.cc:455] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.504884 2363 sys_catalog.cc:458] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: This master's current role is: FOLLOWER
14:58:30.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.505721 2351 sys_catalog.cc:564] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: configured and running, proceeding with master startup.
14:58:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.509057 2177 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: No bootstrap required, opened a new log
14:58:30.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.534857 2177 raft_consensus.cc:357] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.536283 2177 raft_consensus.cc:383] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:58:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.537266 2177 raft_consensus.cc:738] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca49f183b1bf4edca721f1740b11706a, State: Initialized, Role: FOLLOWER
14:58:30.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.538606 2177 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:30.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.542605 2377 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:30.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.543999 2377 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: This master's current role is: FOLLOWER
14:58:30.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.546008 2177 sys_catalog.cc:564] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: configured and running, proceeding with master startup.
14:58:30.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.547431 2375 catalog_manager.cc:1560] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:58:30.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.547868 2375 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:58:30.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.551988 2131 tablet.cc:2378] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:58:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.536048 2272 master_runner.cc:186] Error getting master registration for 127.2.7.126:43677: OK, instance_id { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" instance_seqno: 1756911505103354 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:58:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.573491 2272 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:58:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.604776 2388 catalog_manager.cc:1560] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:58:30.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.605468 2388 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:58:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.597826 2106 master_runner.cc:186] Error getting master registration for 127.2.7.125:46609: OK, instance_id { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" instance_seqno: 1756911507702922 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:58:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.629137 2106 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:58:30.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:30.745947 2305 tablet.cc:2378] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:58:30.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.750222 2363 raft_consensus.cc:491] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:58:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.750865 2363 raft_consensus.cc:513] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.759307 2152 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7032a980ea84c088d1c1d07dc94a137" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ca49f183b1bf4edca721f1740b11706a" is_pre_election: true
14:58:30.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.760352 2152 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d7032a980ea84c088d1c1d07dc94a137 in term 0.
14:58:30.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.771003 2363 leader_election.cc:290] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677), cc97b3e1a10940c0aad4d42a187be1e5 (127.2.7.125:46609)
14:58:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.772765 2302 leader_election.cc:304] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ca49f183b1bf4edca721f1740b11706a, d7032a980ea84c088d1c1d07dc94a137; no voters:
14:58:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.773804 2363 raft_consensus.cc:2802] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:58:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.774158 2363 raft_consensus.cc:491] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:58:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.774487 2363 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 0 FOLLOWER]: Advancing to term 1
14:58:30.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.783464 2363 raft_consensus.cc:513] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.787634 2152 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7032a980ea84c088d1c1d07dc94a137" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ca49f183b1bf4edca721f1740b11706a"
14:58:30.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.788332 2152 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 0 FOLLOWER]: Advancing to term 1
14:58:30.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.799882 2152 raft_consensus.cc:2466] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d7032a980ea84c088d1c1d07dc94a137 in term 1.
14:58:30.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.800633 2363 leader_election.cc:290] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [CANDIDATE]: Term 1 election: Requested vote from peers ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677), cc97b3e1a10940c0aad4d42a187be1e5 (127.2.7.125:46609)
14:58:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.802481 2302 leader_election.cc:304] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ca49f183b1bf4edca721f1740b11706a, d7032a980ea84c088d1c1d07dc94a137; no voters:
14:58:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.804556 2363 raft_consensus.cc:2802] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Leader election won for term 1
14:58:30.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.854205 2363 raft_consensus.cc:695] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 LEADER]: Becoming Leader. State: Replica: d7032a980ea84c088d1c1d07dc94a137, State: Running, Role: LEADER
14:58:30.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.855643 2363 consensus_queue.cc:237] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.881968 2390 sys_catalog.cc:455] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d7032a980ea84c088d1c1d07dc94a137. Latest consensus state: current_term: 1 leader_uuid: "d7032a980ea84c088d1c1d07dc94a137" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:30.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.882835 2390 sys_catalog.cc:458] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: This master's current role is: LEADER
14:58:30.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.901785 2392 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:58:30.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.939821 2392 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:58:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.987665 2152 raft_consensus.cc:1273] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 1 FOLLOWER]: Refusing update from remote peer d7032a980ea84c088d1c1d07dc94a137: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:58:30.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:30.989526 2390 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:58:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.051048 2377 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: SysCatalogTable state changed. Reason: New leader d7032a980ea84c088d1c1d07dc94a137. Latest consensus state: current_term: 1 leader_uuid: "d7032a980ea84c088d1c1d07dc94a137" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:31.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.052983 2377 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: This master's current role is: FOLLOWER
14:58:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.060673 2363 sys_catalog.cc:455] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d7032a980ea84c088d1c1d07dc94a137" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.061446 2363 sys_catalog.cc:458] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: This master's current role is: LEADER
14:58:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.066705 2392 catalog_manager.cc:1349] Generated new cluster ID: 290ae34b2db24a288391fbea498b77af
14:58:31.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.066999 2392 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:58:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.078580 2377 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d7032a980ea84c088d1c1d07dc94a137" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:31.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.079272 2377 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: This master's current role is: FOLLOWER
14:58:31.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.146591 2392 catalog_manager.cc:1372] Generated new certificate authority record
14:58:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.149188 2392 catalog_manager.cc:1506] Loading token signing keys...
14:58:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.220157 2392 catalog_manager.cc:5955] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Generated new TSK 0
14:58:31.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.228705 2392 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:58:31.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.7.65@KRBTEST.COM; defaulting to no policy
14:58:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:31.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.7.65@KRBTEST.COM" created.
14:58:31.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:31.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:31.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:31.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.7.65@KRBTEST.COM; defaulting to no policy
14:58:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:31.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.7.65@KRBTEST.COM" created.
14:58:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.435302 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/wal
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/logs
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data/info.pb
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.65:0
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.7.65
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.65
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.65
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:31.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.611541 2388 catalog_manager.cc:1261] Loaded cluster ID: 290ae34b2db24a288391fbea498b77af
14:58:31.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.612305 2388 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: loading cluster ID for follower catalog manager: success
14:58:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.631603 2388 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: acquiring CA information for follower catalog manager: success
14:58:31.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:31.637238 2388 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:58:32.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.060242 2402 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:32.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.060914 2402 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.061980 2402 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:32.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.120047 2402 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:32.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.122036 2402 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.7.65
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.192574 2402 tablet_server_runner.cc:78] Tablet server non-default flags:
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/wal
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.65
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.65:0
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data/info.pb
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.65
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:32.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.7.65
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/logs
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:32.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.194375 2402 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:32.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.196131 2402 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.248353 2410 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:32.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.273741 2262 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } attempt: 8
14:58:32.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:32 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911507, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.125@KRBTEST.COM for kudu/127.2.7.124@KRBTEST.COM
14:58:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.417490 2262 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Bootstrap starting.
14:58:32.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.454384 2262 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Neither blocks nor log segments found. Creating new log.
14:58:32.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.462261 2262 log.cc:826] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Log is configured to *not* fsync() on all Append() calls
14:58:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.491051 2262 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: No bootstrap required, opened a new log
14:58:32.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.542529 2302 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.2.7.125:46609 timed out after 1.769s (SENT)
14:58:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.872663 2262 raft_consensus.cc:357] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:32.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.883073 2262 raft_consensus.cc:383] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:58:32.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.883384 2262 raft_consensus.cc:738] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cc97b3e1a10940c0aad4d42a187be1e5, State: Initialized, Role: FOLLOWER
14:58:32.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.894430 2262 consensus_queue.cc:260] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.918676 2262 sys_catalog.cc:564] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: configured and running, proceeding with master startup.
14:58:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.918857 2237 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7032a980ea84c088d1c1d07dc94a137" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" is_pre_election: true
14:58:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.919721 2237 raft_consensus.cc:2466] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d7032a980ea84c088d1c1d07dc94a137 in term 0.
14:58:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.920486 2237 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34178 (request call id 1) took 2146 ms (client timeout 1769 ms). Trace:
14:58:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.921993 2236 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7032a980ea84c088d1c1d07dc94a137" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc97b3e1a10940c0aad4d42a187be1e5"
14:58:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.922554 2236 raft_consensus.cc:3058] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 0 FOLLOWER]: Advancing to term 1
14:58:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.923113 2420 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.923862 2420 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: This master's current role is: FOLLOWER
14:58:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.925526 2233 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7032a980ea84c088d1c1d07dc94a137" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc97b3e1a10940c0aad4d42a187be1e5"
14:58:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.927424 2234 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7032a980ea84c088d1c1d07dc94a137" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" is_pre_election: true
14:58:32.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.920873 2237 rpcz_store.cc:269] 0903 14:58:30.773734 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:58:32.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:30.773901 (+ 167us) service_pool.cc:225] Handling call
14:58:32.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:32.920395 (+2146494us) inbound_call.cc:173] Queueing success response
14:58:32.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:58:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.935789 2236 raft_consensus.cc:2466] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d7032a980ea84c088d1c1d07dc94a137 in term 1.
14:58:32.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.945048 2235 raft_consensus.cc:1273] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Refusing update from remote peer d7032a980ea84c088d1c1d07dc94a137: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:58:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.952056 2235 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:34178 (request call id 3) took 1966 ms. Trace:
14:58:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.953987 2236 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:34178 (request call id 2) took 2166 ms (client timeout 2073 ms). Trace:
14:58:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.954964 2236 rpcz_store.cc:269] 0903 14:58:30.787621 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:58:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:30.787773 (+ 152us) service_pool.cc:225] Handling call
14:58:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:32.953917 (+2166144us) inbound_call.cc:173] Queueing success response
14:58:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
14:58:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.952734 2235 rpcz_store.cc:276] 0903 14:58:30.985559 (+ 0us) service_pool.cc:168] Inserting onto call queue
14:58:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:30.985669 (+ 110us) service_pool.cc:225] Handling call
14:58:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:32.944334 (+1958665us) raft_consensus.cc:1469] Updating replica for 0 ops
14:58:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:32.945504 (+ 1170us) raft_consensus.cc:1695] Filling consensus response to leader.
14:58:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:32.951727 (+ 6223us) spinlock_profiling.cc:243] Waited 17 ms on lock 0x7b5400030550. stack: 00007f099b86e473 00007f099b86e370 00007f099b3e7574 00000000004fcff4 00000000004fcf89 00007f09a277b6e1 00007f09a27f686d 00007f09a37e2c89 00007f09a1f83b44 00007f09a1f83ad2 00007f09a1f83a01 00007f09a1f8397c 00007f09a1f82c32 00007f09a05b6ddc 00007f09a05b6216 00007f09a05b5bbf
14:58:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:32.951982 (+ 255us) inbound_call.cc:173] Queueing success response
14:58:32.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":39149312}
14:58:32.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:32.959252 2419 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [LEADER]: Connected to new peer: Peer: permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:58:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:33.085422 2433 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d7032a980ea84c088d1c1d07dc94a137" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:33.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:33.086261 2433 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: This master's current role is: FOLLOWER
14:58:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:33.086447 2420 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d7032a980ea84c088d1c1d07dc94a137. Latest consensus state: current_term: 1 leader_uuid: "d7032a980ea84c088d1c1d07dc94a137" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:33.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:33.087373 2420 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: This master's current role is: FOLLOWER
14:58:33.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:33.108646 2437 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:58:33.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:33.588910 2419 sys_catalog.cc:455] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d7032a980ea84c088d1c1d07dc94a137" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:33.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:33.589874 2419 sys_catalog.cc:458] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: This master's current role is: LEADER
14:58:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.144948 2437 catalog_manager.cc:1261] Loaded cluster ID: 290ae34b2db24a288391fbea498b77af
14:58:34.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.145252 2437 catalog_manager.cc:1554] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: loading cluster ID for follower catalog manager: success
14:58:34.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.172796 2437 catalog_manager.cc:1576] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: acquiring CA information for follower catalog manager: success
14:58:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.179636 2437 catalog_manager.cc:1604] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:58:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:33.620604 2408 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 2402
14:58:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:34.560139 2408 kernel_stack_watchdog.cc:198] Thread 2402 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 363ms:
14:58:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:58:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:58:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:58:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:58:34.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:32.253731 2409 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:34.574676 2412 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:34.573462 2402 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.319s user 0.681s sys 0.980s
14:58:34.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:34.575461 2402 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.321s user 0.681s sys 0.980s
14:58:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.578629 2402 server_base.cc:1047] running on GCE node
14:58:34.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.580125 2402 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:34.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.585012 2402 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:34.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.586614 2402 hybrid_clock.cc:648] HybridClock initialized: now 1756911514586301 us; error 318 us; skew 500 ppm
14:58:34.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.587775 2402 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:34.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:34 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911514, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:34.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.596830 2402 init.cc:377] Logged in from keytab as kudu/127.2.7.65@KRBTEST.COM (short username kudu)
14:58:34.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.622294 2402 webserver.cc:480] Webserver started at http://127.2.7.65:40795/ using document root <none> and password file <none>
14:58:34.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.623642 2402 fs_manager.cc:362] Metadata directory not provided
14:58:34.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.623953 2402 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:34.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.624508 2402 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.631357 2402 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data/instance:
14:58:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1943cca9c84d48f8bada9ba0a737eef7"
14:58:34.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:34 on dist-test-slave-05c6"
14:58:34.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.633057 2402 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/wal/instance:
14:58:34.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1943cca9c84d48f8bada9ba0a737eef7"
14:58:34.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:34 on dist-test-slave-05c6"
14:58:34.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.643816 2402 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.008s sys 0.001s
14:58:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.651098 2450 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:34.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.652570 2402 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
14:58:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.652940 2402 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/wal
14:58:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1943cca9c84d48f8bada9ba0a737eef7"
14:58:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:34 on dist-test-slave-05c6"
14:58:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.653319 2402 fs_report.cc:389] FS layout report
14:58:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:34.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/wal
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/wal
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data/data
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:34.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:34.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.719753 2402 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:34.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.728216 2402 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:34.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.728813 2402 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:34.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.754029 2402 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:58:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.772897 2402 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:58:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.773521 2402 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.001s
14:58:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.773972 2402 ts_tablet_manager.cc:610] Registered 0 tablets
14:58:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.774441 2402 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:58:34.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:34 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911514, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:34.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:34.939908 2121 init.cc:295] Successfully reacquired a new kerberos TGT
14:58:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.089167 2402 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.65:41111
14:58:35.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.089251 2563 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.65:41111 every 8 connection(s)
14:58:35.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.092937 2402 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/data/info.pb
14:58:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.093730 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2402
14:58:35.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.094161 2077 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-0/wal/instance
14:58:35.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.7.66@KRBTEST.COM; defaulting to no policy
14:58:35.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:35 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911514, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.65@KRBTEST.COM for kudu/127.2.7.124@KRBTEST.COM
14:58:35.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:35.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.7.66@KRBTEST.COM" created.
14:58:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.194628 2564 heartbeater.cc:344] Connected to a master server at 127.2.7.124:35655
14:58:35.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.195266 2564 heartbeater.cc:461] Registering TS with master...
14:58:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.196749 2564 heartbeater.cc:507] Master 127.2.7.124:35655 requested a full tablet report, sending...
14:58:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.201628 2316 ts_manager.cc:194] Registered new tserver with Master: 1943cca9c84d48f8bada9ba0a737eef7 (127.2.7.65:41111)
14:58:35.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.204854 2316 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.7.65@KRBTEST.COM'} at 127.2.7.65:41547
14:58:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:35 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911514, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.65@KRBTEST.COM for kudu/127.2.7.125@KRBTEST.COM
14:58:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:35 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911514, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.65@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.225934 2077 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.2.7.66: real 0.114s user 0.003s sys 0.012s
14:58:35.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.234550 2565 heartbeater.cc:344] Connected to a master server at 127.2.7.125:46609
14:58:35.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.234982 2565 heartbeater.cc:461] Registering TS with master...
14:58:35.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.235749 2565 heartbeater.cc:507] Master 127.2.7.125:46609 requested a full tablet report, sending...
14:58:35.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.239311 2227 ts_manager.cc:194] Registered new tserver with Master: 1943cca9c84d48f8bada9ba0a737eef7 (127.2.7.65:41111)
14:58:35.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.7.66@KRBTEST.COM; defaulting to no policy
14:58:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:35.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.7.66@KRBTEST.COM" created.
14:58:35.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.276502 2569 heartbeater.cc:344] Connected to a master server at 127.2.7.126:43677
14:58:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.276997 2569 heartbeater.cc:461] Registering TS with master...
14:58:35.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.277842 2569 heartbeater.cc:507] Master 127.2.7.126:43677 requested a full tablet report, sending...
14:58:35.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.285310 2141 ts_manager.cc:194] Registered new tserver with Master: 1943cca9c84d48f8bada9ba0a737eef7 (127.2.7.65:41111)
14:58:35.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:35.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:35.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.298524 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/wal
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/logs
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data/info.pb
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:35.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.66:0
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.7.66
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.66
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.66
14:58:35.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:35.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.736635 2578 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.737217 2578 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:35.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.738013 2578 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:35.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.780648 2578 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.781860 2578 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.7.66
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.846571 2578 tablet_server_runner.cc:78] Tablet server non-default flags:
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/wal
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.66
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.66:0
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data/info.pb
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.66
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:35.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.7.66
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/logs
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:35.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.848280 2578 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:35.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:35.849664 2578 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:35.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.867867 2585 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:36.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:36.210722 2564 heartbeater.cc:499] Master 127.2.7.124:35655 was elected leader, sending a full tablet report...
14:58:36.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:36.282431 2173 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae1c0 after lost signal to thread 2107
14:58:36.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:36.283408 2173 debug-util.cc:398] Leaking SignalData structure 0x7b0800071a80 after lost signal to thread 2176
14:58:37.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:37 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911517, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:37.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.540619 2204 init.cc:295] Successfully reacquired a new kerberos TGT
14:58:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:37.268748 2584 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 2578
14:58:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:37.711050 2588 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:37.708912 2578 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.842s user 0.312s sys 0.644s
14:58:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:35.868920 2586 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:37.712066 2578 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.845s user 0.313s sys 0.645s
14:58:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:37.715672 2587 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1844 milliseconds
14:58:37.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.715691 2578 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.717006 2578 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:37.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.720134 2578 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:37.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.721568 2578 hybrid_clock.cc:648] HybridClock initialized: now 1756911517721496 us; error 103 us; skew 500 ppm
14:58:37.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.722467 2578 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:37 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911517, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.729251 2578 init.cc:377] Logged in from keytab as kudu/127.2.7.66@KRBTEST.COM (short username kudu)
14:58:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.736505 2578 webserver.cc:480] Webserver started at http://127.2.7.66:40621/ using document root <none> and password file <none>
14:58:37.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.737589 2578 fs_manager.cc:362] Metadata directory not provided
14:58:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.737828 2578 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:37.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.738349 2578 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.742877 2578 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data/instance:
14:58:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7749584c30ec40788889c0a591dc71fb"
14:58:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:37 on dist-test-slave-05c6"
14:58:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.744305 2578 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/wal/instance:
14:58:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7749584c30ec40788889c0a591dc71fb"
14:58:37.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:37 on dist-test-slave-05c6"
14:58:37.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.764137 2578 fs_manager.cc:696] Time spent creating directory manager: real 0.019s user 0.010s sys 0.011s
14:58:37.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.773205 2600 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.774677 2578 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.002s
14:58:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.775080 2578 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/wal
14:58:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7749584c30ec40788889c0a591dc71fb"
14:58:37.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:37 on dist-test-slave-05c6"
14:58:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.775516 2578 fs_report.cc:389] FS layout report
14:58:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/wal
14:58:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/wal
14:58:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data/data
14:58:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:37.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.868867 2578 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.877902 2578 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:37.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.878355 2578 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.881577 2578 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:58:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.887691 2578 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:58:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.887944 2578 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:58:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.888171 2578 ts_tablet_manager.cc:610] Registered 0 tablets
14:58:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:37.888307 2578 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:58:38.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.131543 2578 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.66:44117
14:58:38.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.131758 2713 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.66:44117 every 8 connection(s)
14:58:38.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.134302 2578 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/data/info.pb
14:58:38.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.137094 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2578
14:58:38.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.137650 2077 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-1/wal/instance
14:58:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.7.67@KRBTEST.COM; defaulting to no policy
14:58:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.7.67@KRBTEST.COM" created.
14:58:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:38 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911517, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.66@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:38 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911517, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.66@KRBTEST.COM for kudu/127.2.7.124@KRBTEST.COM
14:58:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:38 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911517, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.66@KRBTEST.COM for kudu/127.2.7.125@KRBTEST.COM
14:58:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.242453 2716 heartbeater.cc:344] Connected to a master server at 127.2.7.126:43677
14:58:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.242960 2714 heartbeater.cc:344] Connected to a master server at 127.2.7.124:35655
14:58:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.243028 2716 heartbeater.cc:461] Registering TS with master...
14:58:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.243391 2714 heartbeater.cc:461] Registering TS with master...
14:58:38.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.244679 2714 heartbeater.cc:507] Master 127.2.7.124:35655 requested a full tablet report, sending...
14:58:38.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.245455 2715 heartbeater.cc:344] Connected to a master server at 127.2.7.125:46609
14:58:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.245712 2716 heartbeater.cc:507] Master 127.2.7.126:43677 requested a full tablet report, sending...
14:58:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.245831 2715 heartbeater.cc:461] Registering TS with master...
14:58:38.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.246860 2715 heartbeater.cc:507] Master 127.2.7.125:46609 requested a full tablet report, sending...
14:58:38.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.247758 2316 ts_manager.cc:194] Registered new tserver with Master: 7749584c30ec40788889c0a591dc71fb (127.2.7.66:44117)
14:58:38.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.248814 2141 ts_manager.cc:194] Registered new tserver with Master: 7749584c30ec40788889c0a591dc71fb (127.2.7.66:44117)
14:58:38.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.250555 2227 ts_manager.cc:194] Registered new tserver with Master: 7749584c30ec40788889c0a591dc71fb (127.2.7.66:44117)
14:58:38.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.250541 2316 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.7.66@KRBTEST.COM'} at 127.2.7.66:40385
14:58:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.7.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.255963 2077 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.2.7.67: real 0.106s user 0.005s sys 0.010s
14:58:38.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.7.67@KRBTEST.COM; defaulting to no policy
14:58:38.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:38.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.7.67@KRBTEST.COM" created.
14:58:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:58:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:38.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.7.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab.
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.297680 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/wal
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/logs
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data/info.pb
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:38.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.67:0
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.7.67
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.67
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.67
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:38.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.646701 2729 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:38.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.647246 2729 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:38.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.647991 2729 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.680809 2729 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:38.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.681986 2729 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.7.67
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.723173 2729 tablet_server_runner.cc:78] Tablet server non-default flags:
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/wal
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:38.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.67
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.67:0
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data/info.pb
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.67
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.7.67
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/logs
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.724828 2729 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:38.726081 2729 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:38.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.746098 2735 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:39.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:39.254547 2714 heartbeater.cc:499] Master 127.2.7.124:35655 was elected leader, sending a full tablet report...
14:58:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.756373 2736 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:38.759315 2738 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:40.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.070282 2729 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:40.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:40.070900 2737 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:58:40.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.074506 2729 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:40.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.076792 2729 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.078212 2729 hybrid_clock.cc:648] HybridClock initialized: now 1756911520078168 us; error 51 us; skew 500 ppm
14:58:40.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.079166 2729 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:40.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:40 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911520, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:40.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.085628 2729 init.cc:377] Logged in from keytab as kudu/127.2.7.67@KRBTEST.COM (short username kudu)
14:58:40.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.091802 2729 webserver.cc:480] Webserver started at http://127.2.7.67:41689/ using document root <none> and password file <none>
14:58:40.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.092856 2729 fs_manager.cc:362] Metadata directory not provided
14:58:40.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.093098 2729 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:40.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.093645 2729 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:58:40.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.099207 2729 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data/instance:
14:58:40.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bfc1f9899484e098832f27fd196f5f0"
14:58:40.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:40 on dist-test-slave-05c6"
14:58:40.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.100564 2729 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/wal/instance:
14:58:40.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bfc1f9899484e098832f27fd196f5f0"
14:58:40.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:40 on dist-test-slave-05c6"
14:58:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.109233 2729 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
14:58:40.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.116283 2749 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:40.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.117830 2729 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
14:58:40.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.118289 2729 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/wal
14:58:40.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bfc1f9899484e098832f27fd196f5f0"
14:58:40.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:40 on dist-test-slave-05c6"
14:58:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.118884 2729 fs_report.cc:389] FS layout report
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/wal
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/wal
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data/data
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:40.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:40.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:40 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911520, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:40.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.134146 2293 init.cc:295] Successfully reacquired a new kerberos TGT
14:58:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.188987 2729 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:40.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.195708 2729 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:40.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.196239 2729 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:40.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.200246 2729 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:58:40.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.205351 2729 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:58:40.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.205591 2729 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
14:58:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.205885 2729 ts_tablet_manager.cc:610] Registered 0 tablets
14:58:40.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.206054 2729 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:58:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.385517 2729 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.67:35047
14:58:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.385627 2861 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.67:35047 every 8 connection(s)
14:58:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.388202 2729 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/data/info.pb
14:58:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.398846 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2729
14:58:40.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.399435 2077 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/ts-2/wal/instance
14:58:40.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:40 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911520, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.67@KRBTEST.COM for kudu/127.2.7.124@KRBTEST.COM
14:58:40.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:40 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911520, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.67@KRBTEST.COM for kudu/127.2.7.125@KRBTEST.COM
14:58:40.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:40 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911520, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.67@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:40.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.459048 2863 heartbeater.cc:344] Connected to a master server at 127.2.7.125:46609
14:58:40.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.459599 2863 heartbeater.cc:461] Registering TS with master...
14:58:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.460732 2863 heartbeater.cc:507] Master 127.2.7.125:46609 requested a full tablet report, sending...
14:58:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.461439 2864 heartbeater.cc:344] Connected to a master server at 127.2.7.126:43677
14:58:40.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.461782 2864 heartbeater.cc:461] Registering TS with master...
14:58:40.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.462678 2864 heartbeater.cc:507] Master 127.2.7.126:43677 requested a full tablet report, sending...
14:58:40.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.464607 2226 ts_manager.cc:194] Registered new tserver with Master: 2bfc1f9899484e098832f27fd196f5f0 (127.2.7.67:35047)
14:58:40.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.466100 2141 ts_manager.cc:194] Registered new tserver with Master: 2bfc1f9899484e098832f27fd196f5f0 (127.2.7.67:35047)
14:58:40.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.473850 2862 heartbeater.cc:344] Connected to a master server at 127.2.7.124:35655
14:58:40.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.474174 2862 heartbeater.cc:461] Registering TS with master...
14:58:40.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.474972 2862 heartbeater.cc:507] Master 127.2.7.124:35655 requested a full tablet report, sending...
14:58:40.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.476800 2316 ts_manager.cc:194] Registered new tserver with Master: 2bfc1f9899484e098832f27fd196f5f0 (127.2.7.67:35047)
14:58:40.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.478137 2316 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.7.67@KRBTEST.COM'} at 127.2.7.67:53263
14:58:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.483505 2077 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:58:40.494 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:40.495 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:58:40.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:58:40.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:58:40.508 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.7.126"
port: 43677
}
bound_http_address {
host: "127.2.7.126"
port: 42535
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.7.125"
port: 46609
}
bound_http_address {
host: "127.2.7.125"
port: 44041
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.7.124"
port: 35655
}
bound_http_address {
host: "127.2.7.124"
port: 46313
}
}
}
14:58:40.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:58:40.510 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.2.7.65"
port: 41111
}
bound_http_address {
host: "127.2.7.65"
port: 40795
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.2.7.66"
port: 44117
}
bound_http_address {
host: "127.2.7.66"
port: 40621
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.2.7.67"
port: 35047
}
bound_http_address {
host: "127.2.7.67"
port: 41689
}
}
}
14:58:40.510 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:58:40.511 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:58:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:40 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911520, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:58:40.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:40.546 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc
14:58:40.552 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:58:40.553 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc
14:58:40.555 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:58:40.558 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:58:40.560 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:58:40.560 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:58:40.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:40 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911520, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:40.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:58:40.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:40.592 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc
14:58:40.594 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:58:40.594 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.7.125:46609
14:58:40.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:58:40.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.597730 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2186
14:58:40.628 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:40.628 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.7.124:35655
14:58:40.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:58:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.629370 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2272
14:58:40.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:40.657 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.7.126:43677
14:58:40.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:58:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.657908 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2106
14:58:40.689 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:40.689 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.7.125:46609
14:58:40.692 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:58:40.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:40.694646 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:40.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/logs
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/info.pb
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.125:46609
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.125
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44041
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.125
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:58:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.071722 2878 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:41.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.072271 2878 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:41.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.072886 2878 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:41.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.109392 2878 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:58:41.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.109660 2878 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:41.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.109930 2878 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:41.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.110142 2878 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:41.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.110337 2878 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:41.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.110524 2878 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:41.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.110674 2878 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32965/jwks.json
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:41.150748 2878 master_runner.cc:386] Master server non-default flags:
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.125
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.125:46609
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/info.pb
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.125
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44041
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/logs
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:41.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:41.152092 2878 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:41.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:41.153183 2878 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.165212 2884 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.239558 2564 heartbeater.cc:646] Failed to heartbeat to 127.2.7.124:35655 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:41.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.265650 2716 heartbeater.cc:646] Failed to heartbeat to 127.2.7.126:43677 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:41.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.474558 2864 heartbeater.cc:646] Failed to heartbeat to 127.2.7.126:43677 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:41.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:41.481469 2862 heartbeater.cc:499] Master 127.2.7.124:35655 was elected leader, sending a full tablet report...
14:58:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:42.039232 1586 debug-util.cc:398] Leaking SignalData structure 0x7b080006f060 after lost signal to thread 1519
14:58:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:42.039974 1586 debug-util.cc:398] Leaking SignalData structure 0x7b0800086c40 after lost signal to thread 1589
14:58:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.171031 2887 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:42.447683 2886 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1281 milliseconds
14:58:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:41.168416 2885 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:42.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.447768 2878 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:42.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.453116 2878 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:42.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.456905 2878 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.458451 2878 hybrid_clock.cc:648] HybridClock initialized: now 1756911522458315 us; error 142 us; skew 500 ppm
14:58:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.459698 2878 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:42 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911522, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:42.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.469300 2878 init.cc:377] Logged in from keytab as kudu/127.2.7.125@KRBTEST.COM (short username kudu)
14:58:42.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.477675 2878 webserver.cc:480] Webserver started at http://127.2.7.125:44041/ using document root <none> and password file <none>
14:58:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.480227 2878 fs_manager.cc:362] Metadata directory not provided
14:58:42.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.480784 2878 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.494530 2878 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.006s sys 0.000s
14:58:42.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.502363 2910 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.503852 2878 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.003s sys 0.000s
14:58:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.504232 2878 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc97b3e1a10940c0aad4d42a187be1e5"
14:58:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:27 on dist-test-slave-05c6"
14:58:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.506354 2878 fs_report.cc:389] FS layout report
14:58:42.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/wal
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/data
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.557155 2878 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:42.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.571354 2878 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:42.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.571981 2878 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.671216 2878 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.125:46609
14:58:42.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.671236 2961 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.125:46609 every 8 connection(s)
14:58:42.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.675450 2878 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-1/data/info.pb
14:58:42.679 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:42.679 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.7.124:35655
14:58:42.679 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:58:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.678089 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2878
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.681540 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/logs
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/info.pb
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:42.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.124:35655
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.124
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46313
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:42.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.124
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:42.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.690007 2962 sys_catalog.cc:263] Verifying existing consensus state
14:58:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.698882 2962 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Bootstrap starting.
14:58:42.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.753252 2962 log.cc:826] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Log is configured to *not* fsync() on all Append() calls
14:58:42.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.773600 2962 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:58:42.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.774643 2962 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Bootstrap complete.
14:58:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.809983 2962 raft_consensus.cc:357] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:42.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.815238 2962 raft_consensus.cc:738] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: cc97b3e1a10940c0aad4d42a187be1e5, State: Initialized, Role: FOLLOWER
14:58:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.816784 2962 consensus_queue.cc:260] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:42.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.831301 2966 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:42.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.832818 2966 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: This master's current role is: FOLLOWER
14:58:42.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.833721 2962 sys_catalog.cc:564] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: configured and running, proceeding with master startup.
14:58:42.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.925148 2977 catalog_manager.cc:1261] Loaded cluster ID: 290ae34b2db24a288391fbea498b77af
14:58:42.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.925500 2977 catalog_manager.cc:1554] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: loading cluster ID for follower catalog manager: success
14:58:42.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.943650 2977 catalog_manager.cc:1576] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: acquiring CA information for follower catalog manager: success
14:58:42.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.949766 2977 catalog_manager.cc:1604] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:58:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.902248 2878 master_runner.cc:186] Error getting master registration for 127.2.7.126:43677: Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111),
14:58:42.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:42.955129 2878 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:58:43.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.158965 2964 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:43.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.159808 2964 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:43.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.161227 2964 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:43.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.197424 2964 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:58:43.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.197702 2964 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:43.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.198088 2964 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:43.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.198421 2964 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:43.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.199344 2964 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:43.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.199657 2964 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:43.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.199880 2964 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32965/jwks.json
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.240115 2964 master_runner.cc:386] Master server non-default flags:
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.124
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.124:35655
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/info.pb
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.124
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46313
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/logs
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.241799 2964 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:43.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.243263 2964 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:43.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.313211 2989 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:43.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.355144 2565 heartbeater.cc:344] Connected to a master server at 127.2.7.125:46609
14:58:43.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.356464 2715 heartbeater.cc:344] Connected to a master server at 127.2.7.125:46609
14:58:43.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.363596 2927 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "1943cca9c84d48f8bada9ba0a737eef7" instance_seqno: 1756911515035657) as {username='kudu', principal='kudu/127.2.7.65@KRBTEST.COM'} at 127.2.7.65:38463; Asking this server to re-register.
14:58:43.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.364500 2926 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "7749584c30ec40788889c0a591dc71fb" instance_seqno: 1756911518077435) as {username='kudu', principal='kudu/127.2.7.66@KRBTEST.COM'} at 127.2.7.66:60503; Asking this server to re-register.
14:58:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.365753 2715 heartbeater.cc:461] Registering TS with master...
14:58:43.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.366142 2565 heartbeater.cc:461] Registering TS with master...
14:58:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.369462 2926 ts_manager.cc:194] Registered new tserver with Master: 7749584c30ec40788889c0a591dc71fb (127.2.7.66:44117)
14:58:43.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.369626 2927 ts_manager.cc:194] Registered new tserver with Master: 1943cca9c84d48f8bada9ba0a737eef7 (127.2.7.65:41111)
14:58:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.537986 2863 heartbeater.cc:344] Connected to a master server at 127.2.7.125:46609
14:58:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.542901 2927 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "2bfc1f9899484e098832f27fd196f5f0" instance_seqno: 1756911520343071) as {username='kudu', principal='kudu/127.2.7.67@KRBTEST.COM'} at 127.2.7.67:34135; Asking this server to re-register.
14:58:43.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.546991 2863 heartbeater.cc:461] Registering TS with master...
14:58:43.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.552838 2927 ts_manager.cc:194] Registered new tserver with Master: 2bfc1f9899484e098832f27fd196f5f0 (127.2.7.67:35047)
14:58:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:44 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911524, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.602046 2446 init.cc:295] Successfully reacquired a new kerberos TGT
14:58:44.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.621773 3005 raft_consensus.cc:491] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:58:44.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.622395 3005 raft_consensus.cc:513] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:44.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.625368 3005 leader_election.cc:290] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677), d7032a980ea84c088d1c1d07dc94a137 (127.2.7.124:35655)
14:58:44.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:44.629226 2913 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:44.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:44.643194 2913 leader_election.cc:336] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677): Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:44.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:44.647917 2911 leader_election.cc:336] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d7032a980ea84c088d1c1d07dc94a137 (127.2.7.124:35655): Network error: Client connection negotiation failed: client connection to 127.2.7.124:35655: connect: Connection refused (error 111)
14:58:44.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.648402 2911 leader_election.cc:304] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc97b3e1a10940c0aad4d42a187be1e5; no voters: ca49f183b1bf4edca721f1740b11706a, d7032a980ea84c088d1c1d07dc94a137
14:58:44.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.649312 3005 raft_consensus.cc:2747] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:58:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:43.344106 2964 server_base.cc:1047] running on GCE node
14:58:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.346544 2995 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:44.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:43.345894 2990 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:44.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.821704 2964 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:44.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.826807 2964 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.828294 2964 hybrid_clock.cc:648] HybridClock initialized: now 1756911524828115 us; error 190 us; skew 500 ppm
14:58:44.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.829511 2964 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:44 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911524, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:44.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.838588 2964 init.cc:377] Logged in from keytab as kudu/127.2.7.124@KRBTEST.COM (short username kudu)
14:58:44.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.851460 2964 webserver.cc:480] Webserver started at http://127.2.7.124:46313/ using document root <none> and password file <none>
14:58:44.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.852491 2964 fs_manager.cc:362] Metadata directory not provided
14:58:44.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.852742 2964 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:44.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.862103 2964 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
14:58:44.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.867594 3015 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.870783 2964 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.005s sys 0.000s
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.871179 2964 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d7032a980ea84c088d1c1d07dc94a137"
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:30 on dist-test-slave-05c6"
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.873137 2964 fs_report.cc:389] FS layout report
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/wal
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/data
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:44.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.937880 2964 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.946270 2964 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:44.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:44.946868 2964 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:45.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.046280 2964 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.124:35655
14:58:45.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.046415 3066 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.124:35655 every 8 connection(s)
14:58:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.049249 2964 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-2/data/info.pb
14:58:45.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.052742 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 2964
14:58:45.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:45.054 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.7.126:43677
14:58:45.054 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.056576 2077 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskB842dE/build/tsan/bin/kudu
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/logs
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/info.pb
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:58:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.126:43677
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.126
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42535
14:58:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.126
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:58:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:58:45.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.068678 3067 sys_catalog.cc:263] Verifying existing consensus state
14:58:45.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.083056 3067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Bootstrap starting.
14:58:45.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.141130 3067 log.cc:826] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Log is configured to *not* fsync() on all Append() calls
14:58:45.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.164894 3067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:58:45.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.165993 3067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: Bootstrap complete.
14:58:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.200776 3067 raft_consensus.cc:357] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:45.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.204246 3067 raft_consensus.cc:738] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d7032a980ea84c088d1c1d07dc94a137, State: Initialized, Role: FOLLOWER
14:58:45.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.205377 3067 consensus_queue.cc:260] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:45.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.208549 3071 sys_catalog.cc:455] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:45.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.209726 3071 sys_catalog.cc:458] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: This master's current role is: FOLLOWER
14:58:45.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.210875 3067 sys_catalog.cc:564] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: configured and running, proceeding with master startup.
14:58:45.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.260470 3082 catalog_manager.cc:1261] Loaded cluster ID: 290ae34b2db24a288391fbea498b77af
14:58:45.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.260782 3082 catalog_manager.cc:1554] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: loading cluster ID for follower catalog manager: success
14:58:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.268532 3082 catalog_manager.cc:1576] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: acquiring CA information for follower catalog manager: success
14:58:45.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.274075 3082 catalog_manager.cc:1604] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:58:45.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.257992 2964 master_runner.cc:186] Error getting master registration for 127.2.7.126:43677: Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111),
14:58:45.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.286686 2964 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:58:45.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.334890 2564 heartbeater.cc:344] Connected to a master server at 127.2.7.124:35655
14:58:45.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.351274 3032 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "1943cca9c84d48f8bada9ba0a737eef7" instance_seqno: 1756911515035657) as {username='kudu', principal='kudu/127.2.7.65@KRBTEST.COM'} at 127.2.7.65:43337; Asking this server to re-register.
14:58:45.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.358580 2564 heartbeater.cc:461] Registering TS with master...
14:58:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.363354 3032 ts_manager.cc:194] Registered new tserver with Master: 1943cca9c84d48f8bada9ba0a737eef7 (127.2.7.65:41111)
14:58:45.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.381121 2714 heartbeater.cc:344] Connected to a master server at 127.2.7.124:35655
14:58:45.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.384423 3032 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "7749584c30ec40788889c0a591dc71fb" instance_seqno: 1756911518077435) as {username='kudu', principal='kudu/127.2.7.66@KRBTEST.COM'} at 127.2.7.66:36927; Asking this server to re-register.
14:58:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.387689 2714 heartbeater.cc:461] Registering TS with master...
14:58:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.392454 3032 ts_manager.cc:194] Registered new tserver with Master: 7749584c30ec40788889c0a591dc71fb (127.2.7.66:44117)
14:58:45.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.557543 2862 heartbeater.cc:344] Connected to a master server at 127.2.7.124:35655
14:58:45.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.558097 2862 heartbeater.cc:499] Master 127.2.7.124:35655 was elected leader, sending a full tablet report...
14:58:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.560894 3032 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "2bfc1f9899484e098832f27fd196f5f0" instance_seqno: 1756911520343071) as {username='kudu', principal='kudu/127.2.7.67@KRBTEST.COM'} at 127.2.7.67:35253; Asking this server to re-register.
14:58:45.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.563016 2862 heartbeater.cc:461] Registering TS with master...
14:58:45.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.571352 3032 ts_manager.cc:194] Registered new tserver with Master: 2bfc1f9899484e098832f27fd196f5f0 (127.2.7.67:35047)
14:58:45.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.641599 3069 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:58:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.642423 3069 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:58:45.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.643541 3069 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:58:45.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.695834 1586 debug-util.cc:398] Leaking SignalData structure 0x7b080005a1e0 after lost signal to thread 1519
14:58:45.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.696653 1586 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0220 after lost signal to thread 1589
14:58:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.705782 3069 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:58:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.706135 3069 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:58:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.706621 3069 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:58:45.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.706970 3069 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:58:45.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.707309 3069 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:58:45.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.707635 3069 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:58:45.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.707880 3069 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:32965/jwks.json
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.764756 3069 master_runner.cc:386] Master server non-default flags:
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.7.84:33021
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.7.126:43677,127.2.7.125:46609,127.2.7.124:35655
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/krb5kdc/kudu.keytab
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.7.126
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.7.126:43677
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:58:45.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32965/jwks.json
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/info.pb
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.7.126
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42535
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:58:45.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/logs
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 03 Sep 2025 14:39:09 UTC on 5fd53c4cbb9d
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7881
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:58:45.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.766192 3069 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:58:45.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:45.767709 3069 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:58:45.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.782637 3098 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:46.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.736488 3108 raft_consensus.cc:491] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:58:46.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.738616 3108 raft_consensus.cc:513] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:46.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.774219 3108 leader_election.cc:290] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677), cc97b3e1a10940c0aad4d42a187be1e5 (127.2.7.125:46609)
14:58:46.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:46.787204 3018 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:46.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.790843 3110 raft_consensus.cc:491] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:58:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.795948 3110 raft_consensus.cc:513] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.826673 3110 leader_election.cc:290] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677), d7032a980ea84c088d1c1d07dc94a137 (127.2.7.124:35655)
14:58:46.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:46.837535 3018 leader_election.cc:336] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677): Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:46.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.850293 3042 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d7032a980ea84c088d1c1d07dc94a137" is_pre_election: true
14:58:46.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.851495 3042 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate cc97b3e1a10940c0aad4d42a187be1e5 in term 1.
14:58:46.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.854909 2911 leader_election.cc:304] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: cc97b3e1a10940c0aad4d42a187be1e5, d7032a980ea84c088d1c1d07dc94a137; no voters:
14:58:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.856042 3110 raft_consensus.cc:2802] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:58:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.856523 3110 raft_consensus.cc:491] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:58:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.856956 3110 raft_consensus.cc:3058] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 1 FOLLOWER]: Advancing to term 2
14:58:46.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.881304 3110 raft_consensus.cc:513] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:46.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:46.889551 2913 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.2.7.126:43677, user_credentials={real_user=slave}} blocked reactor thread for 45643.4us
14:58:46.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.891579 2937 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d7032a980ea84c088d1c1d07dc94a137" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" is_pre_election: true
14:58:46.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.892594 2937 raft_consensus.cc:2391] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d7032a980ea84c088d1c1d07dc94a137 in current term 2: Already voted for candidate cc97b3e1a10940c0aad4d42a187be1e5 in this term.
14:58:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.894356 3018 leader_election.cc:304] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d7032a980ea84c088d1c1d07dc94a137; no voters: ca49f183b1bf4edca721f1740b11706a, cc97b3e1a10940c0aad4d42a187be1e5
14:58:46.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.895933 3108 raft_consensus.cc:3058] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 1 FOLLOWER]: Advancing to term 2
14:58:46.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:46.905985 2913 leader_election.cc:336] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677): Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:46.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.909611 3110 leader_election.cc:290] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 election: Requested vote from peers ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677), d7032a980ea84c088d1c1d07dc94a137 (127.2.7.124:35655)
14:58:46.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.911118 3042 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "d7032a980ea84c088d1c1d07dc94a137"
14:58:46.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.916092 3108 raft_consensus.cc:2747] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:58:46.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:46.910404 2913 leader_election.cc:336] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677): Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111)
14:58:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:46.942641 3042 raft_consensus.cc:2466] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate cc97b3e1a10940c0aad4d42a187be1e5 in term 2.
14:58:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.013422 2911 leader_election.cc:304] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: cc97b3e1a10940c0aad4d42a187be1e5, d7032a980ea84c088d1c1d07dc94a137; no voters: ca49f183b1bf4edca721f1740b11706a
14:58:47.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.014670 3110 raft_consensus.cc:2802] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 2 FOLLOWER]: Leader election won for term 2
14:58:47.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.035967 3110 raft_consensus.cc:695] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [term 2 LEADER]: Becoming Leader. State: Replica: cc97b3e1a10940c0aad4d42a187be1e5, State: Running, Role: LEADER
14:58:47.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.037817 3110 consensus_queue.cc:237] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.080514 3118 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cc97b3e1a10940c0aad4d42a187be1e5. Latest consensus state: current_term: 2 leader_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:47.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.098577 3118 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: This master's current role is: LEADER
14:58:47.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.115556 3120 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:58:47.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.198943 3120 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:58:47.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.204851 3120 catalog_manager.cc:1261] Loaded cluster ID: 290ae34b2db24a288391fbea498b77af
14:58:47.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.218910 3120 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:58:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.235539 3120 catalog_manager.cc:1506] Loading token signing keys...
14:58:47.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.263269 3120 catalog_manager.cc:5966] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5: Loaded TSK: 0
14:58:47.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.266331 3120 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:58:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:47.635723 2913 consensus_peers.cc:489] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 -> Peer ca49f183b1bf4edca721f1740b11706a (127.2.7.126:43677): Couldn't send request to peer ca49f183b1bf4edca721f1740b11706a. Status: Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:58:47.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.663084 3042 raft_consensus.cc:1273] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [term 2 FOLLOWER]: Refusing update from remote peer cc97b3e1a10940c0aad4d42a187be1e5: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:58:47.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.667274 3118 consensus_queue.cc:1035] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
14:58:47.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.699389 3127 sys_catalog.cc:455] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: SysCatalogTable state changed. Reason: New leader cc97b3e1a10940c0aad4d42a187be1e5. Latest consensus state: current_term: 2 leader_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:47.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.700512 3127 sys_catalog.cc:458] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: This master's current role is: FOLLOWER
14:58:47.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.719810 3127 sys_catalog.cc:455] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:47.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.720544 3127 sys_catalog.cc:458] T 00000000000000000000000000000000 P d7032a980ea84c088d1c1d07dc94a137 [sys.catalog]: This master's current role is: FOLLOWER
14:58:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.723606 3118 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:47.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.724320 3118 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: This master's current role is: LEADER
14:58:47.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:47 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911527, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:47.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.735566 2596 init.cc:295] Successfully reacquired a new kerberos TGT
14:58:47.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.958596 2878 master_runner.cc:186] Error getting master registration for 127.2.7.126:43677: Network error: Client connection negotiation failed: client connection to 127.2.7.126:43677: connect: Connection refused (error 111),
14:58:47.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:47.959232 2878 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:58:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:48.397516 2565 heartbeater.cc:499] Master 127.2.7.125:46609 was elected leader, sending a full tablet report...
14:58:48.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:48.407900 2715 heartbeater.cc:499] Master 127.2.7.125:46609 was elected leader, sending a full tablet report...
14:58:48.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:48.574015 2863 heartbeater.cc:499] Master 127.2.7.125:46609 was elected leader, sending a full tablet report...
14:58:49.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:47.186672 3097 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 3069
14:58:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:49.608665 3097 kernel_stack_watchdog.cc:198] Thread 3069 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
14:58:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:58:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:58:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:58:49.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:58:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:49.609550 3069 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.826s user 1.285s sys 2.056s
14:58:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:45.784689 3099 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:49.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:49.609967 3069 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.827s user 1.285s sys 2.056s
14:58:49.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:49.613695 3101 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:58:49.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:49.617316 3100 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3826 milliseconds
14:58:49.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.617540 3069 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:58:49.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.618907 3069 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:58:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.622037 3069 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:58:49.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.623558 3069 hybrid_clock.cc:648] HybridClock initialized: now 1756911529623457 us; error 115 us; skew 500 ppm
14:58:49.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.624670 3069 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:58:49.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:49 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911529, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.633241 3069 init.cc:377] Logged in from keytab as kudu/127.2.7.126@KRBTEST.COM (short username kudu)
14:58:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.639590 3069 webserver.cc:480] Webserver started at http://127.2.7.126:42535/ using document root <none> and password file <none>
14:58:49.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.640826 3069 fs_manager.cc:362] Metadata directory not provided
14:58:49.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.641141 3069 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:58:49.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.652173 3069 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.001s
14:58:49.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.658221 3153 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:58:49.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.659626 3069 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
14:58:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.660069 3069 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data,/tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca49f183b1bf4edca721f1740b11706a"
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-03 14:58:24 on dist-test-slave-05c6"
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.662871 3069 fs_report.cc:389] FS layout report
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/wal
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/data
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:58:49.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:58:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:58:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:58:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:58:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:58:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:58:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.741127 3069 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:58:49.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.755419 3069 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:58:49.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.756026 3069 kserver.cc:163] Server-wide thread pool size limit: 3276
14:58:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.857537 3069 rpc_server.cc:307] RPC server started. Bound to: 127.2.7.126:43677
14:58:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.857656 3204 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.7.126:43677 every 8 connection(s)
14:58:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.861541 3069 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskB842dE/test-tmp/mini-kudu-cluster7139471296507089813/master-0/data/info.pb
14:58:49.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.869752 2077 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskB842dE/build/tsan/bin/kudu as pid 3069
14:58:49.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:58:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.892876 3205 sys_catalog.cc:263] Verifying existing consensus state
14:58:49.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:49 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911522, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.125@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.907371 3205 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Bootstrap starting.
14:58:49.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:49.996457 3205 log.cc:826] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Log is configured to *not* fsync() on all Append() calls
14:58:50.025 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.7.125:46609(127.2.7.125:46609)] connecting to peer
14:58:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.036651 3205 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:58:50.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.037670 3205 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: Bootstrap complete.
14:58:50.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.074697 3205 raft_consensus.cc:357] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:50.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.078554 3205 raft_consensus.cc:738] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca49f183b1bf4edca721f1740b11706a, State: Initialized, Role: FOLLOWER
14:58:50.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.079756 3205 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } }
14:58:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:50 dist-test-slave-05c6 krb5kdc[2089](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911530, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:58:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.091601 2745 init.cc:295] Successfully reacquired a new kerberos TGT
14:58:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.095782 3211 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.096855 3211 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: This master's current role is: FOLLOWER
14:58:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.098598 3205 sys_catalog.cc:564] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: configured and running, proceeding with master startup.
14:58:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.099566 3180 raft_consensus.cc:3058] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [term 1 FOLLOWER]: Advancing to term 2
14:58:50.133 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.7.124:35655(127.2.7.124:35655)] connecting to peer
14:58:50.137 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.7.126:43677(127.2.7.126:43677)] connecting to peer
14:58:50.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.152516 3211 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: SysCatalogTable state changed. Reason: New leader cc97b3e1a10940c0aad4d42a187be1e5. Latest consensus state: current_term: 2 leader_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:50.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.154415 3211 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: This master's current role is: FOLLOWER
14:58:50.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.153612 3216 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:50.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.155269 3216 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a [sys.catalog]: This master's current role is: FOLLOWER
14:58:50.199 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.7.124:35655(127.2.7.124:35655)] Successfully connected to peer
14:58:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.207144 3227 catalog_manager.cc:1261] Loaded cluster ID: 290ae34b2db24a288391fbea498b77af
14:58:50.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.207516 3227 catalog_manager.cc:1554] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: loading cluster ID for follower catalog manager: success
14:58:50.211 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.7.125:46609(127.2.7.125:46609)] Successfully connected to peer
14:58:50.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.217896 3227 catalog_manager.cc:1576] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: acquiring CA information for follower catalog manager: success
14:58:50.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 03 14:58:50 dist-test-slave-05c6 krb5kdc[2089](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756911529, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.7.126@KRBTEST.COM for kudu/127.2.7.126@KRBTEST.COM
14:58:50.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.227514 3227 catalog_manager.cc:1604] T 00000000000000000000000000000000 P ca49f183b1bf4edca721f1740b11706a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:58:50.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.333967 3139 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ca49f183b1bf4edca721f1740b11706a" member_type: VOTER last_known_addr { host: "127.2.7.126" port: 43677 } } peers { permanent_uuid: "cc97b3e1a10940c0aad4d42a187be1e5" member_type: VOTER last_known_addr { host: "127.2.7.125" port: 46609 } } peers { permanent_uuid: "d7032a980ea84c088d1c1d07dc94a137" member_type: VOTER last_known_addr { host: "127.2.7.124" port: 35655 } } }
14:58:50.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.334975 3139 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc97b3e1a10940c0aad4d42a187be1e5 [sys.catalog]: This master's current role is: LEADER
14:58:50.366 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.7.126:43677(127.2.7.126:43677)] Successfully connected to peer
14:58:50.387 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:58:50.391 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.7.124:35655(127.2.7.124:35655)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:58:50.391 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.7.125:46609(127.2.7.125:46609)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:58:50.401 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.2.7.124:35655: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:58:50.401 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.7.125:46609: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:50.405881 3231 negotiation.cc:337] Failed RPC negotiation. Trace:
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.198384 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:45012 (local address 127.2.7.125:46609)
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.208505 (+ 10121us) server_negotiation.cc:207] Beginning negotiation
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.208547 (+ 42us) server_negotiation.cc:400] Waiting for connection header
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.305615 (+ 97068us) server_negotiation.cc:408] Connection header received
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.364687 (+ 59072us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:58:50.406 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.7.124:35655(127.2.7.124:35655)] handling channelInactive
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.364729 (+ 42us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.364965 (+ 236us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.365105 (+ 140us) server_negotiation.cc:227] Negotiated authn=SASL
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.404751 (+ 39646us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:45012: BlockingRecv error: recv got EOF from 127.0.0.1:45012 (error 108)
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":9337,"thread_start_us":9019,"threads_started":1}
14:58:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:50.406217 3230 negotiation.cc:337] Failed RPC negotiation. Trace:
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.188853 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43880 (local address 127.2.7.124:35655)
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.213433 (+ 24580us) server_negotiation.cc:207] Beginning negotiation
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.213461 (+ 28us) server_negotiation.cc:400] Waiting for connection header
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.306033 (+ 92572us) server_negotiation.cc:408] Connection header received
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.364686 (+ 58653us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.364723 (+ 37us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.365052 (+ 329us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.365352 (+ 300us) server_negotiation.cc:227] Negotiated authn=SASL
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.404234 (+ 38882us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43880: BlockingRecv error: recv got EOF from 127.0.0.1:43880 (error 108)
14:58:50.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23673,"thread_start_us":24275,"threads_started":1}
14:58:50.407 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.7.126:43677(127.2.7.126:43677)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:58:50.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.2.7.126:43677: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:58:50.417 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=530), Traces: [0ms] refreshing cache from master, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.7.125:46609, [238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.7.124:35655, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.7.126:43677, [502ms] Sub RPC ConnectToMaster: received response from server master-127.2.7.124:35655: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [497ms] Sub RPC ConnectToMaster: received response from server master-127.2.7.125:46609: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [513ms] Sub RPC ConnectToMaster: received response from server master-127.2.7.126:43677: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.2.7.125:46609,127.2.7.124:35655,127.2.7.126:43677). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.2.7.125:46609,127.2.7.124:35655,127.2.7.126:43677). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14:58:50.424 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=537), Traces: [0ms] refreshing cache from master, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.7.125:46609, [238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.7.124:35655, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.7.126:43677, [502ms] Sub RPC ConnectToMaster: received response from server master-127.2.7.124:35655: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [497ms] Sub RPC ConnectToMaster: received response from server master-127.2.7.125:46609: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [513ms] Sub RPC ConnectToMaster: received response from server master-127.2.7.126:43677: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
14:58:50.428 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.7.125:46609(127.2.7.125:46609)] handling channelInactive
14:58:50.428 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.7.124:35655(127.2.7.124:35655)] disconnecting while in state TERMINATED
14:58:50.428 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.7.126:43677(127.2.7.126:43677)] handling channelInactive
14:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:50.427019 3232 negotiation.cc:337] Failed RPC negotiation. Trace:
14:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.208439 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:46480 (local address 127.2.7.126:43677)
14:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.213177 (+ 4738us) server_negotiation.cc:207] Beginning negotiation
14:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.213207 (+ 30us) server_negotiation.cc:400] Waiting for connection header
14:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.366617 (+153410us) server_negotiation.cc:408] Connection header received
14:58:50.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.367528 (+ 911us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:58:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.367561 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:58:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.367806 (+ 245us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:58:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.367978 (+ 172us) server_negotiation.cc:227] Negotiated authn=SASL
14:58:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.425031 (+ 57053us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:46480: BlockingRecv error: recv got EOF from 127.0.0.1:46480 (error 108)
14:58:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":3992,"thread_start_us":4255,"threads_started":1}
14:58:50.434 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.7.126:43677(127.2.7.126:43677)] disconnecting while in state TERMINATED
14:58:50.435 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.7.125:46609(127.2.7.125:46609)] disconnecting while in state TERMINATED
14:58:50.439 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:58:50.441 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:58:50.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.444382 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2402
14:58:50.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.478806 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2578
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250903 14:58:50.506947 3232 negotiation.cc:337] Failed RPC negotiation. Trace:
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.456794 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.2.7.66:44191 (local address 127.2.7.126:43677)
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.462038 (+ 5244us) server_negotiation.cc:207] Beginning negotiation
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.462070 (+ 32us) server_negotiation.cc:400] Waiting for connection header
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.462122 (+ 52us) server_negotiation.cc:408] Connection header received
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.462281 (+ 159us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.462309 (+ 28us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.462554 (+ 245us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.462763 (+ 209us) server_negotiation.cc:227] Negotiated authn=CERTIFICATE
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.477702 (+ 14939us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.479785 (+ 2083us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0903 14:58:50.505653 (+ 25868us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.2.7.66:44191: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:58:50.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4611}
14:58:50.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.508882 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2729
14:58:50.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.541774 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 3069
14:58:50.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.566036 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2878
14:58:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:50.598645 2077 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 2964
14:58:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-03T14:58:51Z chronyd exiting
14:58:52.851 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:58:52.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:52.852620 1355 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 1655
14:58:52.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:52.878213 1355 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 1796
14:58:52.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:52.906740 1355 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 1933
14:58:52.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:52.934027 1355 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 1367
14:58:52.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:52.962858 1355 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 1437
14:58:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250903 14:58:52.994040 1355 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskB842dE/build/tsan/bin/kudu with pid 1512
14:58:54.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-03T14:58:54Z chronyd exiting
14:58:55.185 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.2.7.125:46609,127.2.7.124:35655,127.2.7.126:43677). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.2.7.125:46609,127.2.7.124:35655,127.2.7.126:43677). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
14:58:56.486 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
14:58:56.486 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.2.7.125:46609,127.2.7.124:35655,127.2.7.126:43677). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.2.7.125:46609,127.2.7.124:35655,127.2.7.126:43677). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more